From 642508ccf6ad2c966e730809e4a6dbaf1364f381 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:35:08 +0000 Subject: [PATCH 1/23] flake8: follow all F rules --- diff --git a/.flake8 b/.flake8 index 65dba9a..0a8cb0c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,5 @@ [flake8] -select = I,C,F4 -ignore = F +select = I,C,F exclude = .git, __pycache__, tests, diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 08c9544..bc7d76c 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -1369,7 +1369,7 @@ def _import_comps(session, filename, tag, options): if pkg.type == libcomps.PACKAGE_TYPE_CONDITIONAL: pkgopts['requires'] = pkg.requires for k in pkgopts.keys(): - if six.PY2 and isinstance(pkgopts[k], unicode): + if six.PY2 and isinstance(pkgopts[k], unicode): # noqa: F821 pkgopts[k] = str(pkgopts[k]) s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(pkgopts.keys())]) print(" Package: %s: {%s}" % (pkg.name, s_opts)) @@ -1402,7 +1402,7 @@ def _import_comps_alt(session, filename, tag, options): # no cover 3.x if ptype == 'conditional': pkgopts['requires'] = pdata[pkg] for k in pkgopts.keys(): - if six.PY2 and isinstance(pkgopts[k], unicode): + if six.PY2 and isinstance(pkgopts[k], unicode): # noqa: F821 pkgopts[k] = str(pkgopts[k]) s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(pkgopts.keys())]) print(" Package: %s: {%s}" % (pkg, s_opts)) diff --git a/hub/kojihub.py b/hub/kojihub.py index 4eb7f84..55184a8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3041,7 +3041,7 @@ def lookup_name(table, info, strict=False, create=False): q = """SELECT id,name FROM %s WHERE id=%%(info)d""" % table elif isinstance(info, str): q = """SELECT id,name FROM %s WHERE name=%%(info)s""" % table - elif six.PY2 and isinstance(info, unicode): + elif six.PY2 and isinstance(info, unicode): # noqa: F821 info = koji.fixEncoding(info) q = """SELECT id,name FROM %s WHERE name=%%(info)s""" % table else: diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index e615be0..8515a1a 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -395,7 +395,6 @@ def load_config(environ): - all PythonOptions (except ConfigFile) are now deprecated and support for them will disappear in a future version of Koji """ - logger = logging.getLogger("koji") #get our config file(s) cf = environ.get('koji.hub.ConfigFile', '/etc/koji-hub/hub.conf') cfdir = environ.get('koji.hub.ConfigDir', '/etc/koji-hub/hub.conf.d') diff --git a/koji/__init__.py b/koji/__init__.py index 1ddf48b..16720f5 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -521,7 +521,7 @@ def ensuredir(directory): # note: if head is blank, then we've reached the top of a relative path try: os.mkdir(directory) - except OSError as e: + except OSError: # do not thrown when dir already exists (could happen in a race) if not os.path.isdir(directory): # something else must have gone wrong @@ -3458,7 +3458,7 @@ def fix_encoding(value, fallback='iso8859-15', remove_nonprintable=False): # play encoding tricks for py2 strings if six.PY2: - if isinstance(value, unicode): + if isinstance(value, unicode): # noqa: F821 # just convert it to a utf8-encoded str value = value.encode('utf8') elif isinstance(value, str): diff --git a/koji/arch.py b/koji/arch.py index dc1fde8..431c9bc 100644 --- a/koji/arch.py +++ b/koji/arch.py @@ -176,8 +176,6 @@ def getBestArchFromList(archlist, myarch=None): if myarch is None: myarch = canonArch - mybestarch = getBestArch(myarch) - bestarch = getBestArch(myarch) if bestarch != myarch: bestarchchoice = getBestArchFromList(archlist, bestarch) diff --git a/koji/daemon.py b/koji/daemon.py index e7775cc..8bb3a2e 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -398,7 +398,7 @@ class SCM(object): def _run(cmd, chdir=None, fatal=False, log=True, _count=[0]): if globals().get('KOJIKAMID'): #we've been inserted into kojikamid, use its run() - return run(cmd, chdir=chdir, fatal=fatal, log=log) + return run(cmd, chdir=chdir, fatal=fatal, log=log) # noqa: F821 else: append = (_count[0] > 0) _count[0] += 1 diff --git a/koji/xmlrpcplus.py b/koji/xmlrpcplus.py index e24d415..279d3ab 100644 --- a/koji/xmlrpcplus.py +++ b/koji/xmlrpcplus.py @@ -52,7 +52,7 @@ class ExtendedMarshaller(xmlrpc_client.Marshaller): if six.PY2: - ExtendedMarshaller.dispatch[long] = ExtendedMarshaller.dump_int + ExtendedMarshaller.dispatch[long] = ExtendedMarshaller.dump_int # noqa: F821 diff --git a/vm/kojikamid.py b/vm/kojikamid.py index f4f26ae..311b2e6 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -61,8 +61,8 @@ class fakemodule(object): #make parts of the above insert accessible as koji.X koji = fakemodule() -koji.GenericError = GenericError -koji.BuildError = BuildError +koji.GenericError = GenericError # noqa: F821 +koji.BuildError = BuildError # noqa: F821 def encode_int(n): """If n is too large for a 32bit signed, convert it to a string""" @@ -88,9 +88,9 @@ class WindowsBuild(object): else: self.task_opts = {} self.workdir = '/tmp/build' - ensuredir(self.workdir) + ensuredir(self.workdir) # noqa: F821 self.buildreq_dir = os.path.join(self.workdir, 'buildreqs') - ensuredir(self.buildreq_dir) + ensuredir(self.buildreq_dir) # noqa: F821 self.source_dir = None self.spec_dir = None self.patches_dir = None @@ -148,20 +148,20 @@ class WindowsBuild(object): else: self.logger.info('file %s exists', entry) if errors: - raise BuildError('error validating build environment: %s' % \ - ', '.join(errors)) + raise BuildError('error validating build environment: %s' % # noqa: F821 + ', '.join(errors)) def updateClam(self): """update ClamAV virus definitions""" ret, output = run(['/bin/freshclam', '--quiet']) if ret: - raise BuildError('could not update ClamAV database: %s' % output) + raise BuildError('could not update ClamAV database: %s' % output) # noqa: F821 def checkEnv(self): """make the environment is fit for building in""" for tool in ['/bin/freshclam', '/bin/clamscan', '/bin/patch']: if not os.path.isfile(tool): - raise BuildError('%s is missing from the build environment' % tool) + raise BuildError('%s is missing from the build environment' % tool) # noqa: F821 def zipDir(self, rootdir, filename): rootbase = os.path.basename(rootdir) @@ -178,18 +178,18 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" - src_scm = SCM(self.source_url) - self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) + src_scm = SCM(self.source_url) # noqa: F821 + self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) # noqa: F821 self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: - spec_scm = SCM(self.task_opts['winspec']) - self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) + spec_scm = SCM(self.task_opts['winspec']) # noqa: F821 + self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) # noqa: F821 self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) else: self.spec_dir = self.source_dir if 'patches' in self.task_opts: - patch_scm = SCM(self.task_opts['patches']) - self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) + patch_scm = SCM(self.task_opts['patches']) # noqa: F821 + self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) # noqa: F821 self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) self.applyPatches(self.source_dir, self.patches_dir) self.virusCheck(self.workdir) @@ -200,7 +200,7 @@ class WindowsBuild(object): os.path.isfile(os.path.join(patchdir, patch)) and \ patch.endswith('.patch')] if not patches: - raise BuildError('no patches found at %s' % patchdir) + raise BuildError('no patches found at %s' % patchdir) # noqa: F821 patches.sort() for patch in patches: cmd = ['/bin/patch', '--verbose', '-d', sourcedir, '-p1', '-i', os.path.join(patchdir, patch)] @@ -210,9 +210,9 @@ class WindowsBuild(object): """Load build configuration from the spec file.""" specfiles = [spec for spec in os.listdir(self.spec_dir) if spec.endswith('.ini')] if len(specfiles) == 0: - raise BuildError('No .ini file found') + raise BuildError('No .ini file found') # noqa: F821 elif len(specfiles) > 1: - raise BuildError('Multiple .ini files found') + raise BuildError('Multiple .ini files found') # noqa: F821 if six.PY2: conf = SafeConfigParser() @@ -306,7 +306,7 @@ class WindowsBuild(object): """Create the buildroot object on the hub.""" repo_id = self.task_opts.get('repo_id') if not repo_id: - raise BuildError('repo_id must be specified') + raise BuildError('repo_id must be specified') # noqa: F821 self.buildroot_id = self.server.initBuildroot(repo_id, self.platform) def expireBuildroot(self): @@ -316,9 +316,9 @@ class WindowsBuild(object): def fetchFile(self, basedir, buildinfo, fileinfo, brtype): """Download the file from buildreq, at filepath, into the basedir""" destpath = os.path.join(basedir, fileinfo['localpath']) - ensuredir(os.path.dirname(destpath)) + ensuredir(os.path.dirname(destpath)) # noqa: F821 if 'checksum_type' in fileinfo: - checksum_type = CHECKSUM_TYPES[fileinfo['checksum_type']] + checksum_type = CHECKSUM_TYPES[fileinfo['checksum_type']] # noqa: F821 if checksum_type == 'sha1': checksum = hashlib.sha1() elif checksum_type == 'sha256': @@ -326,7 +326,7 @@ class WindowsBuild(object): elif checksum_type == 'md5': checksum = hashlib.md5() else: - raise BuildError('Unknown checksum type %s for %s' % ( + raise BuildError('Unknown checksum type %s for %s' % ( # noqa: F821 checksum_type, os.path.basename(fileinfo['localpath']))) with open(destpath, 'w') as destfile: @@ -345,7 +345,7 @@ class WindowsBuild(object): if 'checksum_type' in fileinfo: digest = checksum.hexdigest() if fileinfo['checksum'] != digest: - raise BuildError('checksum validation failed for %s, %s (computed) != %s (provided)' % \ + raise BuildError('checksum validation failed for %s, %s (computed) != %s (provided)' % # noqa: F821 (destpath, digest, fileinfo['checksum'])) self.logger.info('Retrieved %s (%s bytes, %s: %s)', destpath, offset, checksum_type, digest) else: @@ -361,7 +361,7 @@ class WindowsBuild(object): buildinfo = self.server.getLatestBuild(self.build_tag, buildreq, self.task_opts.get('repo_id')) br_dir = os.path.join(self.buildreq_dir, buildreq, brtype) - ensuredir(br_dir) + ensuredir(br_dir) # noqa: F821 brinfo['dir'] = br_dir brfiles = [] brinfo['files'] = brfiles @@ -438,7 +438,7 @@ class WindowsBuild(object): cmd = ['cmd.exe', '/C', 'C:\\Windows\\Temp\\' + os.path.basename(tmpname)] ret, output = run(cmd, chdir=self.source_dir) if ret: - raise BuildError('build command failed, see build.log for details') + raise BuildError('build command failed, see build.log for details') # noqa: F821 def bashBuild(self): """Do the build: run the execute line(s) with bash""" @@ -470,7 +470,7 @@ class WindowsBuild(object): cmd = ['/bin/bash', '-e', '-x', tmpname] ret, output = run(cmd, chdir=self.source_dir) if ret: - raise BuildError('build command failed, see build.log for details') + raise BuildError('build command failed, see build.log for details') # noqa: F821 def checkBuild(self): """Verify that the build completed successfully.""" @@ -497,13 +497,13 @@ class WindowsBuild(object): errors.append('file %s does not exist' % entry) self.virusCheck(self.workdir) if errors: - raise BuildError('error validating build output: %s' % \ + raise BuildError('error validating build output: %s' % # noqa: F821 ', '.join(errors)) def virusCheck(self, path): """ensure a path is virus free with ClamAV. path should be absolute""" if not path.startswith('/'): - raise BuildError('Invalid path to scan for viruses: ' + path) + raise BuildError('Invalid path to scan for viruses: ' + path) # noqa: F821 run(['/bin/clamscan', '--quiet', '--recursive', path], fatal=True) def gatherResults(self): @@ -555,7 +555,7 @@ def run(cmd, chdir=None, fatal=False, log=True): msg += ', see %s for details' % (os.path.basename(logfd.name)) else: msg += ', output: %s' % output - raise BuildError(msg) + raise BuildError(msg) # noqa: F821 return ret, output def find_net_info(): diff --git a/vm/kojivmd b/vm/kojivmd index c926fa2..c0fa3ff 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -995,7 +995,7 @@ class VMTaskManager(TaskManager): task_info = self.session.getTaskInfo(task['id'], request=True) vm_name = task_info['request'][0] try: - vm = self.libvirt_conn.lookupByName(vm_name) + self.libvirt_conn.lookupByName(vm_name) except libvirt.libvirtError: # if this builder does not have the requested VM, # we can't handle the task From 97cfaa4fcf88987df1adff3d221531380a9cfa91 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:35:08 +0000 Subject: [PATCH 2/23] flake8: follow E265 rule --- diff --git a/builder/kojid b/builder/kojid index 01a2411..cf4d060 100755 --- a/builder/kojid +++ b/builder/kojid @@ -131,7 +131,7 @@ def main(options, session): tm.findHandlers(globals()) tm.findHandlers(vars(koji.tasks)) if options.plugin: - #load plugins + # load plugins pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':')) for name in options.plugin: logger.info('Loading plugin: %s' % name) @@ -192,9 +192,9 @@ class BuildRoot(object): self._new(*args,**kwargs) def _load(self, data): - #manage an existing buildroot + # manage an existing buildroot if isinstance(data, dict): - #assume data already pulled from db + # assume data already pulled from db self.id = data['id'] else: self.id = data @@ -291,7 +291,7 @@ class BuildRoot(object): opts['tag_macros'][macro] = self.config['extra'][key] output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) - #write config + # write config with open(configfile,'w') as fo: fo.write(output) @@ -398,7 +398,7 @@ class BuildRoot(object): """Run mock""" mockpath = getattr(self.options,"mockpath","/usr/bin/mock") cmd = [mockpath, "-r", self.mockcfg] - #if self.options.debug_mock: + # if self.options.debug_mock: # cmd.append('--debug') # TODO: should we pass something like --verbose --trace instead? if 'mock.new_chroot' in self.config['extra']: @@ -495,7 +495,7 @@ class BuildRoot(object): ts_offsets[fname] = position incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger) - #clean up and return exit status of command + # clean up and return exit status of command for (fname, (fd, inode, size, fpath)) in logs.items(): if not fd: continue @@ -507,7 +507,7 @@ class BuildRoot(object): return status[1] else: - #in no case should exceptions propagate past here + # in no case should exceptions propagate past here try: self.session._forget() if workdir: @@ -524,7 +524,7 @@ class BuildRoot(object): os.setreuid(uid,uid) os.execvp(cmd[0],cmd) except: - #diediedie + # diediedie print("Failed to exec mock") print(''.join(traceback.format_exception(*sys.exc_info()))) os._exit(1) @@ -656,9 +656,9 @@ class BuildRoot(object): ts = rpm.TransactionSet() for h in ts.dbMatch(): pkg = koji.get_header_fields(h, fields) - #skip our fake packages + # skip our fake packages if pkg['name'] in ['buildsys-build', 'gpg-pubkey']: - #XXX config + # XXX config continue pkg['payloadhash'] = koji.hex_string(pkg['sigmd5']) del pkg['sigmd5'] @@ -744,9 +744,9 @@ class BuildRoot(object): external_repos = self.session.getExternalRepoList(self.repo_info['tag_id'], event=self.repo_info['create_event']) if not external_repos: - #nothing to do + # nothing to do return - #index external repos by expanded url + # index external repos by expanded url erepo_idx = {} for erepo in external_repos: # substitute $arch in the url with the arch of the repo we're generating @@ -781,7 +781,7 @@ class BuildRoot(object): pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href'] koji.util.rmtree(tmpdir) elif yum_available: - #XXX - cheap hack to get relative paths + # XXX - cheap hack to get relative paths repomdpath = os.path.join(repodir, self.br_arch, 'repodata', 'repomd.xml') with koji.openRemoteFile(repomdpath, **opts) as fo: try: @@ -796,8 +796,8 @@ class BuildRoot(object): relpath = os.path.join(repodir, self.br_arch, pkgorigins) with koji.openRemoteFile(relpath, **opts) as fo: - #at this point we know there were external repos at the create event, - #so there should be an origins file. + # at this point we know there were external repos at the create event, + # so there should be an origins file. origin_idx = {} # don't use 'with GzipFile' as it is not supported on py2.6 fo2 = GzipFile(fileobj=fo, mode='r') @@ -807,7 +807,7 @@ class BuildRoot(object): parts=line.split(None, 2) if len(parts) < 2: continue - #first field is formated by yum as [e:]n-v-r.a + # first field is formated by yum as [e:]n-v-r.a nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % koji.parse_NVRA(parts[0]) origin_idx[nvra] = parts[1] fo2.close() @@ -874,7 +874,7 @@ class BuildRoot(object): class ChainBuildTask(BaseTaskHandler): Methods = ['chainbuild'] - #mostly just waiting on other tasks + # mostly just waiting on other tasks _taskWeight = 0.1 def handler(self, srcs, target, opts=None): @@ -896,7 +896,7 @@ class ChainBuildTask(BaseTaskHandler): raise koji.GenericError('unknown build target: %s' % target) nvrs = [] for n_level, build_level in enumerate(srcs): - #if there are any nvrs to wait on, do so + # if there are any nvrs to wait on, do so if nvrs: task_id = self.session.host.subtask(method='waitrepo', arglist=[target_info['build_tag_name'], None, nvrs], @@ -904,7 +904,7 @@ class ChainBuildTask(BaseTaskHandler): parent=self.id) self.wait(task_id, all=True, failany=True) nvrs = [] - #kick off the builds for this level + # kick off the builds for this level build_tasks = [] for n_src, src in enumerate(build_level): if SCM.is_scm_url(src): @@ -915,11 +915,11 @@ class ChainBuildTask(BaseTaskHandler): build_tasks.append(task_id) else: nvrs.append(src) - #next pass will wait for these + # next pass will wait for these if build_tasks: - #the level could have been all nvrs + # the level could have been all nvrs self.wait(build_tasks, all=True, failany=True) - #see what builds we created in this batch so the next pass can wait for them also + # see what builds we created in this batch so the next pass can wait for them also for build_task in build_tasks: builds = self.session.listBuilds(taskID=build_task) if builds: @@ -929,7 +929,7 @@ class ChainBuildTask(BaseTaskHandler): class BuildTask(BaseTaskHandler): Methods = ['build'] - #we mostly just wait on other tasks + # we mostly just wait on other tasks _taskWeight = 0.2 def handler(self, src, target, opts=None): @@ -949,7 +949,7 @@ class BuildTask(BaseTaskHandler): self.event_id = repo_info['create_event'] else: repo_info = None - #we'll wait for a repo later (self.getRepo) + # we'll wait for a repo later (self.getRepo) self.event_id = None task_info = self.session.getTaskInfo(self.id) target_info = None @@ -959,7 +959,7 @@ class BuildTask(BaseTaskHandler): dest_tag = target_info['dest_tag'] build_tag = target_info['build_tag'] if repo_info is not None: - #make sure specified repo matches target + # make sure specified repo matches target if repo_info['tag_id'] != target_info['build_tag']: raise koji.BuildError('Repo/Target mismatch: %s/%s' \ % (repo_info['tag_name'], target_info['build_tag_name'])) @@ -970,7 +970,7 @@ class BuildTask(BaseTaskHandler): raise koji.GenericError('unknown build target: %s' % target) build_tag = repo_info['tag_id'] if target is None: - #ok, call it skip-tag for the buildroot tag + # ok, call it skip-tag for the buildroot tag self.opts['skip_tag'] = True dest_tag = build_tag else: @@ -978,7 +978,7 @@ class BuildTask(BaseTaskHandler): if not taginfo: raise koji.GenericError('neither tag nor target: %s' % target) dest_tag = taginfo['id'] - #policy checks... + # policy checks... policy_data = { 'user_id' : task_info['owner'], 'source' : src, @@ -991,7 +991,7 @@ class BuildTask(BaseTaskHandler): if not self.opts.get('skip_tag'): policy_data['tag'] = dest_tag #id if not SCM.is_scm_url(src) and not opts.get('scratch'): - #let hub policy decide + # let hub policy decide self.session.host.assertPolicy('build_from_srpm', policy_data) if opts.get('repo_id') is not None: # use of this option is governed by policy @@ -1024,11 +1024,11 @@ class BuildTask(BaseTaskHandler): % (data['name'], target_info['dest_tag_name'])) # TODO - more pre tests archlist = self.getArchList(build_tag, h, extra=extra_arches) - #let the system know about the build we're attempting + # let the system know about the build we're attempting if not self.opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported build_id = self.session.host.initBuild(data) - #(initBuild raises an exception if there is a conflict) + # (initBuild raises an exception if there is a conflict) failany = (self.opts.get('fail_fast', False) or not getattr(self.options, 'build_arch_can_fail', False)) try: @@ -1037,16 +1037,16 @@ class BuildTask(BaseTaskHandler): repo_info['id'], failany=failany) if opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported self.session.host.moveBuildToScratch(self.id,srpm,rpms,logs=logs) else: self.session.host.completeBuild(self.id,build_id,srpm,rpms,brmap,logs=logs) except (SystemExit,ServerExit,KeyboardInterrupt): - #we do not trap these + # we do not trap these raise except: if not self.opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported self.session.host.failBuild(self.id, build_id) # reraise the exception raise @@ -1067,7 +1067,7 @@ class BuildTask(BaseTaskHandler): return src else: raise koji.BuildError('Invalid source specification: %s' % src) - #XXX - other methods? + # XXX - other methods? def getSRPMFromSRPM(self, src, build_tag, repo_id): # rebuild srpm in mock, so it gets correct disttag, rpm version, etc. @@ -1085,7 +1085,7 @@ class BuildTask(BaseTaskHandler): return srpm def getSRPMFromSCM(self, url, build_tag, repo_id): - #TODO - allow different ways to get the srpm + # TODO - allow different ways to get the srpm task_id = self.session.host.subtask(method='buildSRPMFromSCM', arglist=[url, build_tag, {'repo_id': repo_id, 'scratch': self.opts.get('scratch')}], label='srpm', @@ -1100,7 +1100,7 @@ class BuildTask(BaseTaskHandler): return srpm def readSRPMHeader(self, srpm): - #srpm arg should be a path relative to /work + # srpm arg should be a path relative to /work self.logger.debug("Reading SRPM") relpath = "work/%s" % srpm opts = dict([(k, getattr(self.options, k)) for k in ('topurl','topdir')]) @@ -1117,7 +1117,7 @@ class BuildTask(BaseTaskHandler): buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id) arches = buildconfig['arches'] if not arches: - #XXX - need to handle this better + # XXX - need to handle this better raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) tag_archlist = [koji.canonArch(a) for a in arches.split()] self.logger.debug('arches: %s' % arches) @@ -1139,13 +1139,13 @@ class BuildTask(BaseTaskHandler): if excludearch: archlist = [ a for a in archlist if a not in excludearch ] self.logger.debug('archlist after excludearch: %r' % archlist) - #noarch is funny + # noarch is funny if 'noarch' not in excludearch and \ ( 'noarch' in buildarchs or 'noarch' in exclusivearch ): archlist.append('noarch') override = self.opts.get('arch_override') if self.opts.get('scratch') and override: - #only honor override for scratch builds + # only honor override for scratch builds self.logger.debug('arch override: %s' % override) archlist = override.split() archdict = {} @@ -1248,9 +1248,9 @@ class BuildTask(BaseTaskHandler): return srpm,rpms,brmap,logs def tagBuild(self,build_id,dest_tag): - #XXX - need options to skip tagging and to force tagging - #create the tagBuild subtask - #this will handle the "post tests" + # XXX - need options to skip tagging and to force tagging + # create the tagBuild subtask + # this will handle the "post tests" task_id = self.session.host.subtask(method='tagBuild', arglist=[dest_tag,build_id,False,None,True], label='tag', @@ -1279,7 +1279,7 @@ class BaseBuildTask(BaseTaskHandler): (self.id, self.method, ', '.join(tag_arches), ', '.join(host_arches))) return False - #otherwise... + # otherwise... # This is in principle an error condition, but this is not a good place # to fail. Instead we proceed and let the task fail normally. return True @@ -1448,7 +1448,7 @@ class BuildArchTask(BaseBuildTask): ret['brootid'] = broot.id broot.expire() - #Let TaskManager clean up + # Let TaskManager clean up return ret @@ -1525,7 +1525,7 @@ class MavenTask(MultiPlatformTask): raise except: if not self.opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported self.session.host.failBuild(self.id, self.build_id) # reraise the exception raise @@ -1988,7 +1988,7 @@ class WrapperRPMTask(BaseBuildTask): gid = grp.getgrnam('mock')[2] self.chownTree(specdir, uid, gid) - #build srpm + # build srpm self.logger.debug("Running srpm build") buildroot.build_srpm(specfile, specdir, None) @@ -2327,7 +2327,7 @@ class ChainMavenTask(MultiPlatformTask): class TagBuildTask(BaseTaskHandler): Methods = ['tagBuild'] - #XXX - set weight? + # XXX - set weight? def handler(self, tag_id, build_id, force=False, fromtag=None, ignore_success=False): task = self.session.getTaskInfo(self.id) @@ -2336,11 +2336,11 @@ class TagBuildTask(BaseTaskHandler): self.session.getBuild(build_id, strict=True) self.session.getTag(tag_id, strict=True) - #several basic sanity checks have already been run (and will be run - #again when we make the final call). Our job is to perform the more - #computationally expensive 'post' tests. + # several basic sanity checks have already been run (and will be run + # again when we make the final call). Our job is to perform the more + # computationally expensive 'post' tests. - #XXX - add more post tests + # XXX - add more post tests self.session.host.tagBuild(self.id,tag_id,build_id,force=force,fromtag=fromtag) self.session.host.tagNotification(True, tag_id, fromtag, build_id, user_id, ignore_success) except Exception as e: @@ -2376,7 +2376,7 @@ class BuildBaseImageTask(BuildImageTask): target_info = self.session.getBuildTarget(target, strict=True) build_tag = target_info['build_tag'] repo_info = self.getRepo(build_tag) - #check requested arches against build tag + # check requested arches against build tag buildconfig = self.session.getBuildConfig(build_tag) if not buildconfig['arches']: raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) @@ -2475,11 +2475,11 @@ class BuildBaseImageTask(BuildImageTask): results) except (SystemExit,ServerExit,KeyboardInterrupt): - #we do not trap these + # we do not trap these raise except: if not opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported if bld_info: self.session.host.failBuild(self.id, bld_info['id']) # reraise the exception @@ -2512,7 +2512,7 @@ class BuildApplianceTask(BuildImageTask): target_info = self.session.getBuildTarget(target, strict=True) build_tag = target_info['build_tag'] repo_info = self.getRepo(build_tag) - #check requested arch against build tag + # check requested arch against build tag buildconfig = self.session.getBuildConfig(build_tag) if not buildconfig['arches']: raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) @@ -2561,11 +2561,11 @@ class BuildApplianceTask(BuildImageTask): self.session.host.moveImageBuildToScratch(self.id, results) except (SystemExit,ServerExit,KeyboardInterrupt): - #we do not trap these + # we do not trap these raise except: if not opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported if bld_info: self.session.host.failBuild(self.id, bld_info['id']) # reraise the exception @@ -2597,7 +2597,7 @@ class BuildLiveCDTask(BuildImageTask): target_info = self.session.getBuildTarget(target, strict=True) build_tag = target_info['build_tag'] repo_info = self.getRepo(build_tag) - #check requested arch against build tag + # check requested arch against build tag buildconfig = self.session.getBuildConfig(build_tag) if not buildconfig['arches']: raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) @@ -2645,11 +2645,11 @@ class BuildLiveCDTask(BuildImageTask): self.session.host.moveImageBuildToScratch(self.id, results) except (SystemExit,ServerExit,KeyboardInterrupt): - #we do not trap these + # we do not trap these raise except: if not opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported if bld_info: self.session.host.failBuild(self.id, bld_info['id']) # reraise the exception @@ -2683,7 +2683,7 @@ class BuildLiveMediaTask(BuildImageTask): target_info = self.session.getBuildTarget(target, strict=True) build_tag = target_info['build_tag'] repo_info = self.getRepo(build_tag) - #check requested arch against build tag + # check requested arch against build tag buildconfig = self.session.getBuildConfig(build_tag) if not buildconfig['arches']: raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) @@ -2783,11 +2783,11 @@ class BuildLiveMediaTask(BuildImageTask): self.session.host.moveImageBuildToScratch(self.id, results) except (SystemExit, ServerExit, KeyboardInterrupt): - #we do not trap these + # we do not trap these raise except: if not opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported if bld_info: self.session.host.failBuild(self.id, bld_info['id']) # reraise the exception @@ -2953,7 +2953,7 @@ class ImageTask(BaseTaskHandler): baseurl = '%s/%s' % (repopath, arch) self.logger.debug('BASEURL: %s' % baseurl) self.ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'], repo_info['id']))) - #inject url if provided + # inject url if provided if opts.get('install_tree_url'): self.ks.handler.url(url=opts['install_tree_url']) @@ -3285,7 +3285,7 @@ class LiveCDTask(ImageTask): -## livemedia-creator +# livemedia-creator class LiveMediaTask(ImageTask): Methods = ['createLiveMedia'] @@ -3410,7 +3410,7 @@ class LiveMediaTask(ImageTask): '--no-virt', '--resultdir', resultdir, '--project', name, - #'--tmp', '/tmp' + # '--tmp', '/tmp' ] @@ -3508,10 +3508,10 @@ class LiveMediaTask(ImageTask): if not opts.get('scratch'): # TODO - generate list of rpms in image # (getImagePackages doesn't work here) - #hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), + # hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), # cachedir[1:])) imgdata ['rpmlist'] = [] - #broot.markExternalRPMs(hdrlist) + # broot.markExternalRPMs(hdrlist) broot.expire() return imgdata @@ -3666,10 +3666,10 @@ class OzImageTask(BaseTaskHandler): the way we want """ return { - #Oz specific + # Oz specific 'oz_data_dir': os.path.join(self.workdir, 'oz_data'), 'oz_screenshot_dir': os.path.join(self.workdir, 'oz_screenshots'), - #IF specific + # IF specific 'imgdir': os.path.join(self.workdir, 'scratch_images'), 'tmpdir': os.path.join(self.workdir, 'oz-tmp'), 'verbose': True, @@ -4251,7 +4251,7 @@ class BaseImageTask(OzImageTask): } # record the RPMs that were installed if not opts.get('scratch'): - #fields = ('name', 'version', 'release', 'arch', 'epoch', 'size', + # fields = ('name', 'version', 'release', 'arch', 'epoch', 'size', # 'payloadhash', 'buildtime') icicle = xml.dom.minidom.parseString(images['raw']['icicle']) self.logger.debug('ICICLE: %s' % images['raw']['icicle']) @@ -4540,7 +4540,7 @@ class BuildIndirectionImageTask(OzImageTask): bld_info, target_info, bd) except: if not opts.get('scratch'): - #scratch builds do not get imported + # scratch builds do not get imported if bld_info: self.session.host.failBuild(self.id, bld_info['id']) # reraise the exception @@ -4770,7 +4770,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): 'repo_id': repo_id} if self.options.scm_credentials_dir is not None and os.path.isdir(self.options.scm_credentials_dir): rootopts['bind_opts'] = {'dirs' : {self.options.scm_credentials_dir : '/credentials',}} - ## Force internal_dev_setup back to true because bind_opts is used to turn it off + # Force internal_dev_setup back to true because bind_opts is used to turn it off rootopts['internal_dev_setup'] = True br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id)) broot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, **rootopts) @@ -4820,7 +4820,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): # Run spec file sanity checks. Any failures will throw a BuildError self.spec_sanity_checks(spec_file) - #build srpm + # build srpm self.logger.debug("Running srpm build") broot.build_srpm(spec_file, sourcedir, scm.source_cmd) @@ -4841,7 +4841,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): if srpm_name != os.path.basename(srpm): raise koji.BuildError('srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))) - #upload srpm and return + # upload srpm and return self.uploadFile(srpm) brootid = broot.id @@ -4941,7 +4941,7 @@ Status: %(status)s\r server = smtplib.SMTP(self.options.smtphost) if self.options.smtp_user is not None and self.options.smtp_pass is not None: server.login(self.options.smtp_user, self.options.smtp_pass) - #server.set_debuglevel(True) + # server.set_debuglevel(True) server.sendmail(from_addr, recipients, message) server.quit() @@ -5192,9 +5192,9 @@ class NewRepoTask(BaseTaskHandler): for fn in os.listdir(path): if fn != 'groups' and os.path.isfile("%s/%s/pkglist" % (path, fn)): arches.append(fn) - #see if we can find a previous repo to update from - #only shadowbuild tags should start with SHADOWBUILD, their repos are auto - #expired. so lets get the most recent expired tag for newRepo shadowbuild tasks. + # see if we can find a previous repo to update from + # only shadowbuild tags should start with SHADOWBUILD, their repos are auto + # expired. so lets get the most recent expired tag for newRepo shadowbuild tasks. if tinfo['name'].startswith('SHADOWBUILD'): oldrepo_state = koji.REPO_EXPIRED else: @@ -5242,7 +5242,7 @@ class CreaterepoTask(BaseTaskHandler): _taskWeight = 1.5 def handler(self, repo_id, arch, oldrepo): - #arch is the arch of the repo, not the task + # arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) @@ -5253,7 +5253,7 @@ class CreaterepoTask(BaseTaskHandler): if not os.path.isdir(self.repodir): raise koji.GenericError("Repo directory missing: %s" % self.repodir) groupdata = os.path.join(toprepodir, 'groups', 'comps.xml') - #set up our output dir + # set up our output dir self.outdir = '%s/repo' % self.workdir self.datadir = '%s/repodata' % self.outdir pkglist = os.path.join(self.repodir, 'pkglist') @@ -5286,7 +5286,7 @@ class CreaterepoTask(BaseTaskHandler): cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) - #attempt to recycle repodata from last repo + # attempt to recycle repodata from last repo if pkglist and oldrepo and self.options.createrepo_update: # old repo could be from inherited tag, so path needs to be # composed from that tag, not rinfo['tag_name'] @@ -5459,7 +5459,7 @@ class createDistRepoTask(BaseTaskHandler): "sparc64", "s390x": "s390", "ppc64": "ppc"} def handler(self, tag, repo_id, arch, keys, opts): - #arch is the arch of the repo, not the task + # arch is the arch of the repo, not the task self.rinfo = self.session.repoInfo(repo_id, strict=True) if self.rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo) @@ -6047,7 +6047,7 @@ enabled=1 class WaitrepoTask(BaseTaskHandler): Methods = ['waitrepo'] - #mostly just waiting + # mostly just waiting _taskWeight = 0.2 PAUSE = 60 @@ -6101,7 +6101,7 @@ class WaitrepoTask(BaseTaskHandler): (koji.util.duration(start), taginfo['name'])) return repo else: - #no check requested -- return first ready repo + # no check requested -- return first ready repo return repo if (time.time() - start) > (self.TIMEOUT * 60.0): @@ -6140,7 +6140,7 @@ def get_options(): parser.add_option("--debug-xmlrpc", action="store_true", default=False, help="show xmlrpc debug output") parser.add_option("--debug-mock", action="store_true", default=False, - #obsolete option + # obsolete option help=SUPPRESS_HELP) parser.add_option("--skip-main", action="store_true", default=False, help="don't actually run main") @@ -6163,7 +6163,7 @@ def get_options(): if args: parser.error("incorrect number of arguments") - #not reached + # not reached assert False # pragma: no cover # load local config @@ -6256,12 +6256,12 @@ def get_options(): if getattr(options, name, None) is None: setattr(options, name, value) - #honor topdir + # honor topdir if options.topdir: koji.BASEDIR = options.topdir koji.pathinfo.topdir = options.topdir - #make sure workdir exists + # make sure workdir exists if not os.path.exists(options.workdir): koji.ensuredir(options.workdir) @@ -6308,7 +6308,7 @@ def quit(msg=None, code=1): if __name__ == "__main__": koji.add_file_logger("koji", "/var/log/kojid.log") - #note we're setting logging params for all of koji* + # note we're setting logging params for all of koji* options = get_options() if options.log_level: lvl = getattr(logging, options.log_level, None) @@ -6326,7 +6326,7 @@ if __name__ == "__main__": if options.admin_emails: koji.add_mail_logger("koji", options.admin_emails) - #start a session and login + # start a session and login session_opts = koji.grab_session_options(options) session = koji.ClientSession(options.server, session_opts) if options.cert and os.path.isfile(options.cert): @@ -6360,14 +6360,14 @@ if __name__ == "__main__": quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1]) else: quit("No username/password supplied and Kerberos missing or not configured") - #make session exclusive + # make session exclusive try: session.exclusiveSession(force=options.force_lock) except koji.AuthLockError: quit("Error: Unable to get lock. Trying using --force-lock") if not session.logged_in: quit("Error: Unknown login error") - #make sure it works + # make sure it works try: ret = session.echo("OK") except requests.exceptions.ConnectionError: @@ -6377,7 +6377,7 @@ if __name__ == "__main__": # run main if options.daemon: - #detach + # detach koji.daemonize() main(options, session) # not reached diff --git a/builder/mergerepos b/builder/mergerepos index 7628a78..008dbf4 100755 --- a/builder/mergerepos +++ b/builder/mergerepos @@ -164,7 +164,7 @@ class RepoMerge(object): n = self.yumbase.add_enable_repo(rid, baseurls=[r]) n._merge_rank = count - #setup our sacks + # setup our sacks self.yumbase._getSacks(archlist=self.archlist) self.sort_and_filter() @@ -205,8 +205,8 @@ class RepoMerge(object): if reponum == 0 and not pkg.basepath: # this is the first repo (i.e. the koji repo) and appears # to be using relative urls - #XXX - kind of a hack, but yum leaves us little choice - #force the pkg object to report a relative location + # XXX - kind of a hack, but yum leaves us little choice + # force the pkg object to report a relative location loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) pkg._return_remote_location = make_const_func(loc) if pkg.sourcerpm in seen_srpms: @@ -296,8 +296,8 @@ class RepoMerge(object): if reponum == 0 and not pkg.basepath: # this is the first repo (i.e. the koji repo) and appears # to be using relative urls - #XXX - kind of a hack, but yum leaves us little choice - #force the pkg object to report a relative location + # XXX - kind of a hack, but yum leaves us little choice + # force the pkg object to report a relative location loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) pkg._return_remote_location = make_const_func(loc) diff --git a/cli/koji b/cli/koji index fb9e105..f97e4de 100755 --- a/cli/koji +++ b/cli/koji @@ -50,7 +50,7 @@ def register_plugin(plugin): """ for v in six.itervalues(vars(plugin)): if isinstance(v, six.class_types): - #skip classes + # skip classes continue if callable(v): if getattr(v, 'exported_cli', False): @@ -166,12 +166,12 @@ def get_options(): value = os.path.expanduser(getattr(options, name)) setattr(options, name, value) - #honor topdir + # honor topdir if options.topdir: koji.BASEDIR = options.topdir koji.pathinfo.topdir = options.topdir - #pkgurl is obsolete + # pkgurl is obsolete if options.pkgurl: if options.topurl: warn("Warning: the pkgurl option is obsolete") diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index bc7d76c..f23c8a1 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -484,11 +484,11 @@ def handle_build(options, session, args): opts[key] = val priority = None if build_opts.background: - #relative to koji.PRIO_DEFAULT + # relative to koji.PRIO_DEFAULT priority = 5 # try to check that source is an SRPM if '://' not in source: - #treat source as an srpm and upload it + # treat source as an srpm and upload it if not build_opts.quiet: print("Uploading srpm: %s" % source) serverdir = unique_path('cli-build') @@ -546,7 +546,7 @@ def handle_chain_build(options, session, args): src_list = [] build_level = [] - #src_lists is a list of lists of sources to build. + # src_lists is a list of lists of sources to build. # each list is block of builds ("build level") which must all be completed # before the next block begins. Blocks are separated on the command line with ':' for src in sources: @@ -571,7 +571,7 @@ def handle_chain_build(options, session, args): priority = None if build_opts.background: - #relative to koji.PRIO_DEFAULT + # relative to koji.PRIO_DEFAULT priority = 5 task_id = session.chainBuild(src_list, target, priority=priority) @@ -671,7 +671,7 @@ def handle_maven_build(options, session, args): opts['skip_tag'] = True priority = None if build_opts.background: - #relative to koji.PRIO_DEFAULT + # relative to koji.PRIO_DEFAULT priority = 5 task_id = session.mavenBuild(source, target, opts, priority=priority) if not build_opts.quiet: @@ -894,7 +894,7 @@ def anon_handle_mock_config(goptions, session, args): (options, args) = parser.parse_args(args) activate_session(session, goptions) if args: - #for historical reasons, we also accept buildroot name as first arg + # for historical reasons, we also accept buildroot name as first arg if not options.name: options.name = args[0] else: @@ -1155,7 +1155,7 @@ def handle_import(goptions, session, args): if data['sourcepackage']: break else: - #no srpm included, check for build + # no srpm included, check for build binfo = session.getBuild(nvr) if not binfo: print(_("Missing build or srpm: %s") % nvr) @@ -1164,7 +1164,7 @@ def handle_import(goptions, session, args): print(_("Aborting import")) return - #local function to help us out below + # local function to help us out below def do_import(path, data): rinfo = dict([(k,data[k]) for k in ('name','version','release','arch')]) prev = session.getRPM(rinfo) @@ -1391,13 +1391,13 @@ def _import_comps_alt(session, filename, tag, options): # no cover 3.x uservisible=bool(group.user_visible), description=group.description, langonly=group.langonly) - #yum.comps does not support the biarchonly field + # yum.comps does not support the biarchonly field for ptype, pdata in [('mandatory', group.mandatory_packages), ('default', group.default_packages), ('optional', group.optional_packages), ('conditional', group.conditional_packages)]: for pkg in pdata: - #yum.comps does not support basearchonly + # yum.comps does not support basearchonly pkgopts = {'type' : ptype} if ptype == 'conditional': pkgopts['requires'] = pdata[pkg] @@ -1407,8 +1407,8 @@ def _import_comps_alt(session, filename, tag, options): # no cover 3.x s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(pkgopts.keys())]) print(" Package: %s: {%s}" % (pkg, s_opts)) session.groupPackageListAdd(tag, group.groupid, pkg, force=force, **pkgopts) - #yum.comps does not support group dependencies - #yum.comps does not support metapkgs + # yum.comps does not support group dependencies + # yum.comps does not support metapkgs def handle_import_sig(goptions, session, args): @@ -1540,9 +1540,9 @@ def handle_prune_signed_copies(options, session, args): # 4) for a specified tag, remove all signed copies (no inheritance) # (but skip builds that are multiply tagged) - #for now, we're just implementing mode #1 - #(with the modification that we check to see if the build was latest within - #the last N days) + # for now, we're just implementing mode #1 + # (with the modification that we check to see if the build was latest within + # the last N days) if options.ignore_tag_file: with open(options.ignore_tag_file) as fo: options.ignore_tag.extend([line.strip() for line in fo.readlines()]) @@ -1579,7 +1579,7 @@ def handle_prune_signed_copies(options, session, args): print("...got %i builds" % len(builds)) builds.sort() else: - #single build + # single build binfo = session.getBuild(options.build) if not binfo: parser.error('No such build: %s' % options.build) @@ -1601,21 +1601,21 @@ def handle_prune_signed_copies(options, session, args): time_str = time.asctime(time.localtime(ts)) return "%s: %s" % (time_str, fmt % x) for nvr, binfo in builds: - #listBuilds returns slightly different data than normal + # listBuilds returns slightly different data than normal if 'id' not in binfo: binfo['id'] = binfo['build_id'] if 'name' not in binfo: binfo['name'] = binfo['package_name'] if options.debug: print("DEBUG: %s" % nvr) - #see how recently this build was latest for a tag + # see how recently this build was latest for a tag is_latest = False is_protected = False last_latest = None tags = {} for entry in session.queryHistory(build=binfo['id'])['tag_listing']: - #we used queryHistory rather than listTags so we can consider tags - #that the build was recently untagged from + # we used queryHistory rather than listTags so we can consider tags + # that the build was recently untagged from tags.setdefault(entry['tag.name'], 1) if options.debug: print("Tags: %s" % to_list(tags.keys())) @@ -1633,43 +1633,43 @@ def handle_prune_signed_copies(options, session, args): break if ignore_tag: continue - #in order to determine how recently this build was latest, we have - #to look at the tagging history. + # in order to determine how recently this build was latest, we have + # to look at the tagging history. hist = session.queryHistory(tag=tag_name, package=binfo['name'])['tag_listing'] if not hist: - #really shouldn't happen + # really shouldn't happen raise koji.GenericError("No history found for %s in %s" % (nvr, tag_name)) timeline = [] for x in hist: - #note that for revoked entries, we're effectively splitting them into - #two parts: creation and revocation. + # note that for revoked entries, we're effectively splitting them into + # two parts: creation and revocation. timeline.append((x['create_event'], 1, x)) - #at the same event, revokes happen first + # at the same event, revokes happen first if x['revoke_event'] is not None: timeline.append((x['revoke_event'], 0, x)) timeline.sort(key=lambda entry: entry[:2]) - #find most recent creation entry for our build and crop there + # find most recent creation entry for our build and crop there latest_ts = None for i in range(len(timeline)-1, -1, -1): - #searching in reverse cronological order + # searching in reverse cronological order event_id, is_create, entry = timeline[i] if entry['build_id'] == binfo['id'] and is_create: latest_ts = event_id break if not latest_ts: - #really shouldn't happen + # really shouldn't happen raise koji.GenericError("No creation event found for %s in %s" % (nvr, tag_name)) our_entry = entry if options.debug: print(_histline(event_id, our_entry)) - #now go through the events since most recent creation entry + # now go through the events since most recent creation entry timeline = timeline[i+1:] if not timeline: is_latest = True if options.debug: print("%s is latest in tag %s" % (nvr, tag_name)) break - #before we go any further, is this a protected tag? + # before we go any further, is this a protected tag? protect_tag = False for pattern in options.protect_tag: if fnmatch.fnmatch(tag_name, pattern): @@ -1680,13 +1680,13 @@ def handle_prune_signed_copies(options, session, args): # if this build was in this tag within that limit, then we will # not prune its signed copies if our_entry['revoke_event'] is None: - #we're still tagged with a protected tag + # we're still tagged with a protected tag if options.debug: print("Build %s has protected tag %s" % (nvr, tag_name)) is_protected = True break elif our_entry['revoke_ts'] > cutoff_ts: - #we were still tagged here sometime before the cutoff + # we were still tagged here sometime before the cutoff if options.debug: print("Build %s had protected tag %s until %s" \ % (nvr, tag_name, time.asctime(time.localtime(our_entry['revoke_ts'])))) @@ -1696,40 +1696,40 @@ def handle_prune_signed_copies(options, session, args): revoke_ts = None others = {} for event_id, is_create, entry in timeline: - #So two things can knock this build from the title of latest: + # So two things can knock this build from the title of latest: # - it could be untagged (entry revoked) # - another build could become latest (replaced) - #Note however that if the superceding entry is itself revoked, then - #our build could become latest again + # Note however that if the superceding entry is itself revoked, then + # our build could become latest again if options.debug: print(_histline(event_id, entry)) if entry['build_id'] == binfo['id']: if is_create: - #shouldn't happen + # shouldn't happen raise koji.GenericError("Duplicate creation event found for %s in %s" \ % (nvr, tag_name)) else: - #we've been revoked + # we've been revoked revoke_ts = entry['revoke_ts'] break else: if is_create: - #this build has become latest + # this build has become latest replaced_ts = entry['create_ts'] if entry['active']: - #this entry not revoked yet, so we're done for this tag + # this entry not revoked yet, so we're done for this tag break - #since this entry is revoked later, our build might eventually be - #uncovered, so we have to keep looking + # since this entry is revoked later, our build might eventually be + # uncovered, so we have to keep looking others[entry['build_id']] = 1 else: - #other build revoked - #see if our build has resurfaced + # other build revoked + # see if our build has resurfaced if entry['build_id'] in others: del others[entry['build_id']] if replaced_ts is not None and not others: - #we've become latest again - #(note: we're not revoked yet because that triggers a break above) + # we've become latest again + # (note: we're not revoked yet because that triggers a break above) replaced_ts = None latest_ts = entry['revoke_ts'] if last_latest is None: @@ -1738,25 +1738,25 @@ def handle_prune_signed_copies(options, session, args): timestamps = [last_latest] if revoke_ts is None: if replaced_ts is None: - #turns out we are still latest + # turns out we are still latest is_latest = True if options.debug: print("%s is latest (again) in tag %s" % (nvr, tag_name)) break else: - #replaced (but not revoked) + # replaced (but not revoked) timestamps.append(replaced_ts) if options.debug: print("tag %s: %s not latest (replaced %s)" \ % (tag_name, nvr, time.asctime(time.localtime(replaced_ts)))) elif replaced_ts is None: - #revoked but not replaced + # revoked but not replaced timestamps.append(revoke_ts) if options.debug: print("tag %s: %s not latest (revoked %s)" \ % (tag_name, nvr, time.asctime(time.localtime(revoke_ts)))) else: - #revoked AND replaced + # revoked AND replaced timestamps.append(min(revoke_ts, replaced_ts)) if options.debug: print("tag %s: %s not latest (revoked %s, replaced %s)" \ @@ -1772,13 +1772,13 @@ def handle_prune_signed_copies(options, session, args): continue if is_protected: continue - #not latest anywhere since cutoff, so we can remove all signed copies + # not latest anywhere since cutoff, so we can remove all signed copies rpms = session.listRPMs(buildID=binfo['id']) session.multicall = True for rpminfo in rpms: session.queryRPMSigs(rpm_id=rpminfo['id']) by_sig = {} - #index by sig + # index by sig for rpminfo, [sigs] in zip(rpms, session.multiCall()): for sig in sigs: sigkey = sig['sigkey'] @@ -1799,7 +1799,7 @@ def handle_prune_signed_copies(options, session, args): except OSError: continue if not stat.S_ISREG(st.st_mode): - #warn about this + # warn about this print("Skipping %s. Not a regular file" % signedpath) continue if st.st_mtime > cutoff_ts: @@ -1819,7 +1819,7 @@ def handle_prune_signed_copies(options, session, args): mycount +=1 build_files += 1 build_space += st.st_size - #XXX - this makes some layout assumptions, but + # XXX - this makes some layout assumptions, but # pathinfo doesn't report what we need mydir = os.path.dirname(signedpath) archdirs[mydir] = 1 @@ -2078,7 +2078,7 @@ def handle_list_signed(goptions, session, args): for rinfo in rpms: rpm_idx.setdefault(rinfo['id'], rinfo) tagged[rinfo['id']] = 1 - #Now figure out which sig entries actually have live copies + # Now figure out which sig entries actually have live copies for sig in sigs: rpm_id = sig['rpm_id'] sigkey = sig['sigkey'] @@ -2862,7 +2862,7 @@ def anon_handle_list_pkgs(goptions, session, args): # no limiting clauses were specified allpkgs = True opts['inherited'] = not options.noinherit - #hiding dups only makes sense if we're querying a tag + # hiding dups only makes sense if we're querying a tag if options.tag: opts['with_dups'] = options.show_dups else: @@ -3736,7 +3736,7 @@ def handle_add_target(goptions, session, args): if len(args) > 2: dest_tag = args[2] else: - #most targets have the same name as their destination + # most targets have the same name as their destination dest_tag = name activate_session(session, goptions) if not (session.hasPerm('admin') or session.hasPerm('target')): @@ -3866,7 +3866,7 @@ def anon_handle_list_targets(goptions, session, args): targets = [x[1] for x in tmp_list] for target in targets: print(fmt % target) - #pprint.pprint(session.getBuildTargets()) + # pprint.pprint(session.getBuildTargets()) def _printInheritance(tags, sibdepths=None, reverse=False): @@ -3992,7 +3992,7 @@ def anon_handle_list_tags(goptions, session, args): tags = session.listTags(buildinfo.get('id',None), pkginfo.get('id',None)) tags.sort(key=lambda x: x['name']) - #if options.verbose: + # if options.verbose: # fmt = "%(name)s [%(id)i] %(perm)s %(locked)s %(arches)s" if options.show_id: fmt = "%(name)s [%(id)i]" @@ -4094,14 +4094,14 @@ def _print_histline(entry, **kwargs): if len(edit) != 1: bad_edit = "%i elements" % (len(edit)+1) other = edit[0] - #check edit for sanity + # check edit for sanity if create or not other[2]: bad_edit = "out of order" if event_id != other[0]: bad_edit = "non-matching" if bad_edit: print("Warning: unusual edit at event %i in table %s (%s)" % (event_id, table, bad_edit)) - #we'll simply treat them as separate events + # we'll simply treat them as separate events pprint.pprint(entry) pprint.pprint(edit) _print_histline(entry, **kwargs) @@ -4415,11 +4415,11 @@ def anon_handle_list_history(goptions, session, args): if x['revoke_event'] is not None: if distinguish_match(x, 'revoked'): timeline.append((x['revoke_event'], table, 0, x.copy())) - #pprint.pprint(timeline[-1]) + # pprint.pprint(timeline[-1]) if distinguish_match(x, 'created'): timeline.append((x['create_event'], table, 1, x)) timeline.sort(key=lambda entry: entry[:3]) - #group edits together + # group edits together new_timeline = [] last_event = None edit_index = {} @@ -4892,7 +4892,7 @@ def handle_edit_tag(goptions, session, args): opts['extra'] = extra if options.remove_extra: opts['remove_extra'] = options.remove_extra - #XXX change callname + # XXX change callname session.editTag2(tag, **opts) @@ -4927,7 +4927,7 @@ def handle_lock_tag(goptions, session, args): selected = [session.getTag(name, strict=True) for name in args] for tag in selected: if options.master: - #set the master lock + # set the master lock if tag['locked']: print(_("Tag %s: master lock already set") % tag['name']) continue @@ -5293,12 +5293,12 @@ def anon_handle_list_external_repos(goptions, session, args): def _pick_external_repo_priority(session, tag): """pick priority after current ones, leaving space for later insertions""" repolist = session.getTagExternalRepos(tag_info=tag) - #ordered by priority + # ordered by priority if not repolist: priority = 5 else: priority = (repolist[-1]['priority'] + 7) // 5 * 5 - #at least 3 higher than current max and a multiple of 5 + # at least 3 higher than current max and a multiple of 5 return priority @@ -5404,7 +5404,7 @@ def handle_remove_external_repo(goptions, session, args): return 0 tags = current_tags if delete: - #removing entirely + # removing entirely if current_tags and not options.force: print(_("Error: external repo %s used by tag(s): %s") % (repo, ', '.join(current_tags))) print(_("Use --force to remove anyway")) @@ -5708,10 +5708,10 @@ def _build_image_indirection(options, task_opts, session, args): if not options.quiet: print("Created task: %d" % task_id) print("Task info: %s/taskinfo?taskID=%s" % (options.weburl, task_id)) - #if task_opts.wait or (task_opts.wait is None and not _running_in_bg()): + # if task_opts.wait or (task_opts.wait is None and not _running_in_bg()): # session.logout() # return watch_tasks(session, [task_id], quiet=options.quiet) - #else: + # else: # return @@ -6045,7 +6045,7 @@ def handle_win_build(options, session, args): opts[key] = val priority = None if build_opts.background: - #relative to koji.PRIO_DEFAULT + # relative to koji.PRIO_DEFAULT priority = 5 task_id = session.winBuild(vm_name, scmurl, target, opts, priority=priority) if not build_opts.quiet: @@ -6376,7 +6376,7 @@ def handle_tag_build(opts, session, args): tasks = [] for pkg in args[1:]: task_id = session.tagBuild(args[0], pkg, force=options.force) - #XXX - wait on task + # XXX - wait on task tasks.append(task_id) print("Created task %d" % task_id) if _running_in_bg() or options.nowait: @@ -6468,7 +6468,7 @@ def handle_untag_build(goptions, session, args): builds = [] for binfo in tagged: if binfo['name'] not in seen_pkg: - #latest for this package + # latest for this package if options.verbose: print(_("Leaving latest build for package %(name)s: %(nvr)s") % binfo) else: diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index f6fceba..4d3213c 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -72,7 +72,7 @@ def arg_filter(arg): pass if arg in ARGMAP: return ARGMAP[arg] - #handle lists/dicts? + # handle lists/dicts? return arg @@ -148,7 +148,7 @@ class TaskWatcher(object): self.level = level self.quiet = quiet - #XXX - a bunch of this stuff needs to adapt to different tasks + # XXX - a bunch of this stuff needs to adapt to different tasks def str(self): if self.info: @@ -189,7 +189,7 @@ class TaskWatcher(object): sys.exit(1) state = self.info['state'] if last: - #compare and note status changes + # compare and note status changes laststate = last['state'] if laststate != state: if not self.quiet: @@ -555,7 +555,7 @@ def activate_session(session, options): noauth = options.authtype == "noauth" or getattr(options, 'noauth', False) runas = getattr(options, 'runas', None) if noauth: - #skip authentication + # skip authentication pass elif options.authtype == "ssl" or os.path.isfile(options.cert) and options.authtype is None: # authenticate using SSL client cert @@ -626,7 +626,7 @@ def _list_tasks(options, session): tasklist = session.listTasks(callopts, qopts) tasks = dict([(x['id'], x) for x in tasklist]) - #thread the tasks + # thread the tasks for t in tasklist: if t['parent'] is not None: parent = tasks.get(t['parent']) diff --git a/hub/kojihub.py b/hub/kojihub.py index 55184a8..b9cba9f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -128,8 +128,8 @@ class Task(object): if host_id is None: return False task_id = self.id - #getting a row lock on this task to ensure task assignment sanity - #no other concurrent transaction should be altering this row + # getting a row lock on this task to ensure task assignment sanity + # no other concurrent transaction should be altering this row q = """SELECT state,host_id FROM task WHERE id=%(task_id)s FOR UPDATE""" r = _fetchSingle(q, locals()) if not r: @@ -153,7 +153,7 @@ class Task(object): if user_id is None: return False task_id = self.id - #getting a row lock on this task to ensure task state sanity + # getting a row lock on this task to ensure task state sanity q = """SELECT owner FROM task WHERE id=%(task_id)s FOR UPDATE""" r = _fetchSingle(q, locals()) if not r: @@ -172,8 +172,8 @@ class Task(object): info = self.getInfo(request=True) self.runCallbacks('preTaskStateChange', info, 'state', koji.TASK_STATES[newstate]) self.runCallbacks('preTaskStateChange', info, 'host_id', host_id) - #we use row-level locks to keep things sane - #note the SELECT...FOR UPDATE + # we use row-level locks to keep things sane + # note the SELECT...FOR UPDATE task_id = self.id if not force: q = """SELECT state,host_id FROM task WHERE id=%(task_id)i FOR UPDATE""" @@ -192,15 +192,15 @@ class Task(object): % (task_id)) return False elif otherhost != host_id: - #task is assigned to someone else + # task is assigned to someone else return False - #otherwise the task is assigned to host_id, so keep going + # otherwise the task is assigned to host_id, so keep going else: if otherhost is None: log_error("Error: task %i is non-free but unlocked (state %i)" % (task_id, state)) return False - #if we reach here, task is either + # if we reach here, task is either # - free and unlocked # - assigned to host_id # - force option is enabled @@ -360,8 +360,8 @@ class Task(object): _dml(update, locals()) self.runCallbacks('postTaskStateChange', info, 'state', koji.TASK_STATES['CANCELED']) self.runCallbacks('postTaskStateChange', info, 'completion_ts', now) - #cancel associated builds (only if state is 'BUILDING') - #since we check build state, we avoid loops with cancel_build on our end + # cancel associated builds (only if state is 'BUILDING') + # since we check build state, we avoid loops with cancel_build on our end b_building = koji.BUILD_STATES['BUILDING'] q = """SELECT id FROM build WHERE task_id = %(task_id)i AND state = %(b_building)i @@ -369,7 +369,7 @@ class Task(object): for (build_id,) in _fetchMulti(q, locals()): cancel_build(build_id, cancel_task=False) if recurse: - #also cancel child tasks + # also cancel child tasks self.cancelChildren() return True @@ -392,7 +392,7 @@ class Task(object): if parent is not None: if strict: raise koji.GenericError("Task %d is not top-level (parent=%d)" % (task_id, parent)) - #otherwise, find the top-level task and go from there + # otherwise, find the top-level task and go from there seen = {task_id:1} while parent is not None: if parent in seen: @@ -401,15 +401,15 @@ class Task(object): seen[task_id] = 1 parent = _singleValue(q, locals()) return Task(task_id).cancelFull(strict=True) - #We handle the recursion ourselves, since self.cancel will stop at - #canceled or closed tasks. + # We handle the recursion ourselves, since self.cancel will stop at + # canceled or closed tasks. tasklist = [task_id] seen = {} - #query for use in loop + # query for use in loop q_children = """SELECT id FROM task WHERE parent = %(task_id)i""" for task_id in tasklist: if task_id in seen: - #shouldn't happen + # shouldn't happen raise koji.GenericError("Task LOOP at task %i" % task_id) seen[task_id] = 1 Task(task_id).cancel(recurse=False) @@ -527,14 +527,14 @@ def make_task(method, arglist, **opts): pdata = dict(zip(fields, r)) if pdata['state'] != koji.TASK_STATES['OPEN']: raise koji.GenericError("Parent task (id %(parent)s) is not open" % opts) - #default to a higher priority than parent + # default to a higher priority than parent opts.setdefault('priority', pdata['priority'] - 1) for f in ('owner', 'arch'): opts.setdefault(f, pdata[f]) opts.setdefault('label', None) else: opts.setdefault('priority', koji.PRIO_DEFAULT) - #calling function should enforce priority limitations, if applicable + # calling function should enforce priority limitations, if applicable opts.setdefault('arch', 'noarch') if not context.session.logged_in: raise koji.GenericError('task must have an owner') @@ -542,7 +542,7 @@ def make_task(method, arglist, **opts): opts['owner'] = context.session.user_id opts['label'] = None opts['parent'] = None - #determine channel from policy + # determine channel from policy policy_data = {} policy_data['method'] = method for key in 'arch', 'parent', 'label', 'owner': @@ -662,7 +662,7 @@ def readGlobalInheritance(event=None): ORDER BY priority """ % (",".join(fields), eventCondition(event)) c.execute(q, locals()) - #convert list of lists into a list of dictionaries + # convert list of lists into a list of dictionaries return [dict(zip(fields, x)) for x in c.fetchall()] def readInheritanceData(tag_id, event=None): @@ -673,7 +673,7 @@ def readInheritanceData(tag_id, event=None): ORDER BY priority """ % (",".join(fields), eventCondition(event)) c.execute(q, locals()) - #convert list of lists into a list of dictionaries + # convert list of lists into a list of dictionaries data = [dict(zip(fields, x)) for x in c.fetchall()] # include the current tag_id as child_id, so we can retrace the inheritance chain later for datum in data: @@ -688,7 +688,7 @@ def readDescendantsData(tag_id, event=None): ORDER BY priority """ % (",".join(fields), eventCondition(event)) c.execute(q, locals()) - #convert list of lists into a list of dictionaries + # convert list of lists into a list of dictionaries data = [dict(zip(fields, x)) for x in c.fetchall()] return data @@ -733,7 +733,7 @@ def _writeInheritanceData(tag_id, changes, clear=False): elif not orig or clear: data[parent_id] = link else: - #not a delete request and we have a previous link to parent + # not a delete request and we have a previous link to parent for f in fields: if orig[f] != link[f]: data[parent_id] = link @@ -752,7 +752,7 @@ def _writeInheritanceData(tag_id, changes, clear=False): # nothing to do log_error("No inheritance changes") return - #check for duplicate priorities + # check for duplicate priorities pri_index = {} for link in six.itervalues(data): if link.get('delete link'): @@ -761,7 +761,7 @@ def _writeInheritanceData(tag_id, changes, clear=False): for pri, dups in six.iteritems(pri_index): if len(dups) <= 1: continue - #oops, duplicate entries for a single priority + # oops, duplicate entries for a single priority dup_ids = [link['parent_id'] for link in dups] raise koji.GenericError("Inheritance priorities must be unique (pri %s: %r )" % (pri, dup_ids)) for parent_id, link in six.iteritems(data): @@ -799,8 +799,8 @@ def readFullInheritance(tag_id, event=None, reverse=False, stops=None, jumps=Non def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdepth, maxdepth, noconfig, pfilter, reverse, jumps): if maxdepth is not None and maxdepth < 1: return - #note: maxdepth is relative to where we are, but currdepth is absolute from - #the top. + # note: maxdepth is relative to where we are, but currdepth is absolute from + # the top. currdepth += 1 top = top.copy() top[tag_id] = 1 @@ -816,11 +816,11 @@ def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdept if id in jumps: id = jumps[id] if id in top: - #LOOP! + # LOOP! if event is None: # only log if the issue is current log_error("Warning: INHERITANCE LOOP detected at %s -> %s, pruning" % (tag_id, id)) - #auto prune + # auto prune continue if id in prunes: # ignore pruned tags @@ -829,16 +829,16 @@ def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdept # ignore intransitive inheritance links, except at root continue if link['priority'] < 0: - #negative priority indicates pruning, rather than inheritance + # negative priority indicates pruning, rather than inheritance prunes[id] = 1 continue if reverse: - #maxdepth logic is different in this case. no propagation + # maxdepth logic is different in this case. no propagation if link['maxdepth'] is not None and link['maxdepth'] < currdepth - 1: continue nextdepth = None else: - #propagate maxdepth + # propagate maxdepth nextdepth = link['maxdepth'] if nextdepth is None: if maxdepth is not None: @@ -847,7 +847,7 @@ def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdept nextdepth = min(nextdepth, maxdepth) - 1 link['nextdepth'] = nextdepth link['currdepth'] = currdepth - #propagate noconfig and pkg_filter controls + # propagate noconfig and pkg_filter controls if link['noconfig']: noconfig = True filter = list(pfilter) # copy @@ -857,10 +857,10 @@ def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdept link['filter'] = filter # check history to avoid redundant entries if id in hist: - #already been there - #BUT, options may have been different + # already been there + # BUT, options may have been different rescan = True - #since rescans are possible, we might have to consider more than one previous hit + # since rescans are possible, we might have to consider more than one previous hit for previous in hist[id]: sufficient = True # is previous sufficient? # if last depth was less than current, then previous insufficient @@ -941,8 +941,8 @@ def pkglist_add(taginfo, pkginfo, owner=None, block=None, extra_arches=None, for def _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, update, policy=False): """Like pkglist_add, but without policy or access check""" - #access control comes a little later (via an assert_policy) - #should not make any changes until after policy is checked + # access control comes a little later (via an assert_policy) + # should not make any changes until after policy is checked tag = get_tag(taginfo, strict=True) tag_id = tag['id'] pkg = lookup_package(pkginfo, strict=False) @@ -959,7 +959,7 @@ def _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, if policy: context.session.assertLogin() policy_data = {'tag' : tag_id, 'action' : action, 'package' : pkginfo, 'force' : force} - #don't check policy for admins using force + # don't check policy for admins using force if not (force and context.session.hasPerm('admin')): assert_policy('package_list', policy_data) if not pkg: @@ -981,11 +981,11 @@ def _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, if previous is None: block = bool(block) if update and not force: - #if update flag is true, require that there be a previous entry + # if update flag is true, require that there be a previous entry raise koji.GenericError("cannot update: tag %s has no data for package %s" \ % (tag['name'], pkg['name'])) else: - #already there (possibly via inheritance) + # already there (possibly via inheritance) if owner is None: owner = previous['owner_id'] changed_owner = previous['owner_id'] != owner @@ -995,14 +995,14 @@ def _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, block = bool(block) if extra_arches is None: extra_arches = previous['extra_arches'] - #see if the data is the same + # see if the data is the same for key, value in (('blocked', block), ('extra_arches', extra_arches)): if previous[key] != value: changed = True break if not changed and not changed_owner and not force: - #no point in adding it again with the same data + # no point in adding it again with the same data return if previous['blocked'] and not block and not force: raise koji.GenericError("package %s is blocked in tag %s" % (pkg['name'], tag['name'])) @@ -1038,7 +1038,7 @@ def _direct_pkglist_remove(taginfo, pkginfo, force=False, policy=False): if policy: context.session.assertLogin() policy_data = {'tag' : tag['id'], 'action' : 'remove', 'package' : pkg['id'], 'force' : force} - #don't check policy for admins using force + # don't check policy for admins using force if not (force and context.session.hasPerm('admin')): assert_policy('package_list', policy_data) user = get_user(context.session.user_id) @@ -1067,7 +1067,7 @@ def pkglist_unblock(taginfo, pkginfo, force=False): pkg = lookup_package(pkginfo, strict=True) context.session.assertLogin() policy_data = {'tag' : tag['id'], 'action' : 'unblock', 'package' : pkg['id'], 'force' : force} - #don't check policy for admins using force + # don't check policy for admins using force if not (force and context.session.hasPerm('admin')): assert_policy('package_list', policy_data) user = get_user(context.session.user_id) @@ -1084,10 +1084,10 @@ def pkglist_unblock(taginfo, pkginfo, force=False): if previous['tag_id'] != tag_id: _pkglist_add(tag_id, pkg_id, previous['owner_id'], False, previous['extra_arches']) else: - #just remove the blocking entry + # just remove the blocking entry _pkglist_remove(tag_id, pkg_id) - #it's possible this was the only entry in the inheritance or that the next entry - #back is also a blocked entry. if so, we need to add it back as unblocked + # it's possible this was the only entry in the inheritance or that the next entry + # back is also a blocked entry. if so, we need to add it back as unblocked pkglist = readPackageList(tag_id, pkgID=pkg_id, inherit=True) if pkg_id not in pkglist or pkglist[pkg_id]['blocked']: _pkglist_add(tag_id, pkg_id, previous['owner_id'], False, previous['extra_arches']) @@ -1174,7 +1174,7 @@ def readPackageList(tagID=None, userID=None, pkgID=None, event=None, inherit=Fal for p in _multiRow(q, locals(), [pair[1] for pair in fields]): pkgid = p['package_id'] if not with_dups and pkgid in packages: - #previous data supercedes + # previous data supercedes continue # apply package filters skip = False @@ -1279,11 +1279,11 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, if inherit: taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] - #regardless of inherit setting, we need to use inheritance to read the - #package list + # regardless of inherit setting, we need to use inheritance to read the + # package list packages = readPackageList(tagID=tag, event=event, inherit=True, pkgID=package) - #these values are used for each iteration + # these values are used for each iteration fields = [('tag.id', 'tag_id'), ('tag.name', 'tag_name'), ('build.id', 'id'), ('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), @@ -1344,7 +1344,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, builds = [] seen = {} # used to enforce the 'latest' option for tagid in taglist: - #log_error(koji.db._quoteparams(q,locals())) + # log_error(koji.db._quoteparams(q,locals())) for build in _multiRow(q, locals(), [pair[1] for pair in fields]): pkgid = build['package_id'] pinfo = packages.get(pkgid, None) @@ -1377,15 +1377,15 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late """ taglist = [tag] if inherit: - #XXX really should cache this - it gets called several places + # XXX really should cache this - it gets called several places # (however, it is fairly quick) taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) - #index builds + # index builds build_idx = dict([(b['build_id'], b) for b in builds]) - #the following query is run for each tag in the inheritance + # the following query is run for each tag in the inheritance fields = [('rpminfo.name', 'name'), ('rpminfo.version', 'version'), ('rpminfo.release', 'release'), @@ -1432,17 +1432,17 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late def _iter_rpms(): for tagid in taglist: if tagid in tags_seen: - #certain inheritance trees can (legitimately) have the same tag - #appear more than once (perhaps once with a package filter and once - #without). The hard part of that was already done by readTaggedBuilds. - #We only need consider each tag once. Note how we use build_idx below. - #(Without this, we could report the same rpm twice) + # certain inheritance trees can (legitimately) have the same tag + # appear more than once (perhaps once with a package filter and once + # without). The hard part of that was already done by readTaggedBuilds. + # We only need consider each tag once. Note how we use build_idx below. + # (Without this, we could report the same rpm twice) continue else: tags_seen[tagid] = 1 query.values['tagid'] = tagid for rpminfo in query.iterate(): - #note: we're checking against the build list because + # note: we're checking against the build list because # it has been filtered by the package list. The tag # tools should endeavor to keep tag_listing sane w.r.t. # the package list, but if there is disagreement the package @@ -1451,7 +1451,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late if build is None: continue elif build['tag_id'] != tagid: - #wrong tag + # wrong tag continue yield rpminfo return [_iter_rpms(), builds] @@ -1469,16 +1469,16 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True """ taglist = [tag] if inherit: - #XXX really should cache this - it gets called several places + # XXX really should cache this - it gets called several places # (however, it is fairly quick) taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] # If type == 'maven', we require that both the build *and* the archive have Maven metadata builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, type=type) - #index builds + # index builds build_idx = dict([(b['build_id'], b) for b in builds]) - #the following query is run for each tag in the inheritance + # the following query is run for each tag in the inheritance fields = [('archiveinfo.id', 'id'), ('archiveinfo.type_id', 'type_id'), ('archiveinfo.btype_id', 'btype_id'), @@ -1527,17 +1527,17 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True tags_seen = {} for tagid in taglist: if tagid in tags_seen: - #certain inheritance trees can (legitimately) have the same tag - #appear more than once (perhaps once with a package filter and once - #without). The hard part of that was already done by readTaggedBuilds. - #We only need consider each tag once. Note how we use build_idx below. - #(Without this, we could report the same rpm twice) + # certain inheritance trees can (legitimately) have the same tag + # appear more than once (perhaps once with a package filter and once + # without). The hard part of that was already done by readTaggedBuilds. + # We only need consider each tag once. Note how we use build_idx below. + # (Without this, we could report the same rpm twice) continue else: tags_seen[tagid] = 1 query.values = {'tagid': tagid, 'package': package} for archiveinfo in query.execute(): - #note: we're checking against the build list because + # note: we're checking against the build list because # it has been filtered by the package list. The tag # tools should endeavor to keep tag_listing sane w.r.t. # the package list, but if there is disagreement the package @@ -1546,7 +1546,7 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True if build is None: continue elif build['tag_id'] != tagid: - #wrong tag + # wrong tag continue archives.append(archiveinfo) return [archives, builds] @@ -1600,7 +1600,7 @@ def _tag_build(tag, build, user_id=None, force=False): else: # use the user associated with the current session user = get_user(context.session.user_id, strict=True) - #access check + # access check assert_tag_access(tag['id'], user_id=user_id, force=force) return _direct_tag_build(tag, build, user, force) @@ -1623,19 +1623,19 @@ def _direct_tag_build(tag, build, user, force=False): query = QueryProcessor(columns=['build_id'], tables=[table], clauses=('active = TRUE',)+clauses, values=locals(), opts={'rowlock':True}) - #note: tag_listing is unique on (build_id, tag_id, active) + # note: tag_listing is unique on (build_id, tag_id, active) if query.executeOne(): - #already tagged + # already tagged if not force: raise koji.TagError("build %s already tagged (%s)" % (nvr, tag['name'])) - #otherwise we retag + # otherwise we retag retag = True if retag: - #revoke the old tag first + # revoke the old tag first update = UpdateProcessor(table, values=locals(), clauses=clauses) update.make_revoke(user_id=user_id) update.execute() - #tag the package + # tag the package insert = InsertProcessor(table) insert.set(tag_id=tag_id, build_id=build_id) insert.make_create(user_id=user_id) @@ -1687,7 +1687,7 @@ def _direct_untag_build(tag, build, user, strict=True, force=False): def grplist_add(taginfo, grpinfo, block=False, force=False, **opts): """Add to (or update) group list for tag""" - #only admins.... + # only admins.... context.session.assertPerm('tag') _grplist_add(taginfo, grpinfo, block, force, **opts) @@ -1702,13 +1702,13 @@ def _grplist_add(taginfo, grpinfo, block, force, **opts): previous = groups.get(group['id'], None) cfg_fields = ('exported', 'display_name', 'is_default', 'uservisible', 'description', 'langonly', 'biarchonly', 'blocked') - #prevent user-provided opts from doing anything strange + # prevent user-provided opts from doing anything strange opts = dslice(opts, cfg_fields, strict=False) if previous is not None: - #already there (possibly via inheritance) + # already there (possibly via inheritance) if previous['blocked'] and not force: raise koji.GenericError("group %s is blocked in tag %s" % (group['name'], tag['name'])) - #check for duplication and grab old data for defaults + # check for duplication and grab old data for defaults changed = False for field in cfg_fields: old = previous[field] @@ -1718,9 +1718,9 @@ def _grplist_add(taginfo, grpinfo, block, force, **opts): else: opts[field] = old if not changed: - #no point in adding it again with the same data + # no point in adding it again with the same data return - #provide available defaults and sanity check data + # provide available defaults and sanity check data opts.setdefault('display_name', group['name']) opts.setdefault('biarchonly', False) opts.setdefault('exported', True) @@ -1728,12 +1728,12 @@ def _grplist_add(taginfo, grpinfo, block, force, **opts): # XXX ^^^ opts['tag_id'] = tag['id'] opts['group_id'] = group['id'] - #revoke old entry (if present) + # revoke old entry (if present) update = UpdateProcessor('group_config', values=opts, clauses=['group_id=%(group_id)s', 'tag_id=%(tag_id)s']) update.make_revoke() update.execute() - #add new entry + # add new entry insert = InsertProcessor('group_config', data=opts) insert.make_create() insert.execute() @@ -1755,7 +1755,7 @@ def grplist_remove(taginfo, grpinfo, force=False): Really this shouldn't be used except in special cases Most of the time you really want to use the block or unblock functions """ - #only admins.... + # only admins.... context.session.assertPerm('tag') _grplist_remove(taginfo, grpinfo, force) @@ -1826,7 +1826,7 @@ def _grplist_unblock(taginfo, grpinfo): def grp_pkg_add(taginfo, grpinfo, pkg_name, block=False, force=False, **opts): """Add package to group for tag""" - #only admins.... + # only admins.... context.session.assertPerm('tag') _grp_pkg_add(taginfo, grpinfo, pkg_name, block, force, **opts) @@ -1845,14 +1845,14 @@ def _grp_pkg_add(taginfo, grpinfo, pkg_name, block, force, **opts): raise koji.GenericError("group %s is blocked in tag %s" % (group['name'], tag['name'])) previous = grp_cfg['packagelist'].get(pkg_name, None) cfg_fields = ('type', 'basearchonly', 'requires') - #prevent user-provided opts from doing anything strange + # prevent user-provided opts from doing anything strange opts = dslice(opts, cfg_fields, strict=False) if previous is not None: - #already there (possibly via inheritance) + # already there (possibly via inheritance) if previous['blocked'] and not force: raise koji.GenericError("package %s blocked in group %s, tag %s" \ % (pkg_name, group['name'], tag['name'])) - #check for duplication and grab old data for defaults + # check for duplication and grab old data for defaults changed = False for field in cfg_fields: old = previous[field] @@ -1862,23 +1862,23 @@ def _grp_pkg_add(taginfo, grpinfo, pkg_name, block, force, **opts): else: opts[field] = old if block: - #from condition above, either previous is not blocked or force is on, - #either way, we should add the entry + # from condition above, either previous is not blocked or force is on, + # either way, we should add the entry changed = True if not changed and not force: - #no point in adding it again with the same data (unless force is on) + # no point in adding it again with the same data (unless force is on) return opts.setdefault('type', 'mandatory') opts['group_id'] = group['id'] opts['tag_id'] = tag['id'] opts['package'] = pkg_name opts['blocked'] = block - #revoke old entry (if present) + # revoke old entry (if present) update = UpdateProcessor('group_package_listing', values=opts, clauses=['group_id=%(group_id)s', 'tag_id=%(tag_id)s', 'package=%(package)s']) update.make_revoke() update.execute() - #add new entry + # add new entry insert = InsertProcessor('group_package_listing', data=opts) insert.make_create() insert.execute() @@ -1890,7 +1890,7 @@ def grp_pkg_remove(taginfo, grpinfo, pkg_name, force=False): Really this shouldn't be used except in special cases Most of the time you really want to use the block or unblock functions """ - #only admins.... + # only admins.... context.session.assertPerm('tag') _grp_pkg_remove(taginfo, grpinfo, pkg_name, force) @@ -1949,7 +1949,7 @@ def _grp_pkg_unblock(taginfo, grpinfo, pkg_name): def grp_req_add(taginfo, grpinfo, reqinfo, block=False, force=False, **opts): """Add group requirement to group for tag""" - #only admins.... + # only admins.... context.session.assertPerm('tag') _grp_req_add(taginfo, grpinfo, reqinfo, block, force, **opts) @@ -1969,14 +1969,14 @@ def _grp_req_add(taginfo, grpinfo, reqinfo, block, force, **opts): raise koji.GenericError("group %s is blocked in tag %s" % (group['name'], tag['name'])) previous = grp_cfg['grouplist'].get(req['id'], None) cfg_fields = ('type', 'is_metapkg') - #prevent user-provided opts from doing anything strange + # prevent user-provided opts from doing anything strange opts = dslice(opts, cfg_fields, strict=False) if previous is not None: - #already there (possibly via inheritance) + # already there (possibly via inheritance) if previous['blocked'] and not force: raise koji.GenericError("requirement on group %s blocked in group %s, tag %s" \ % (req['name'], group['name'], tag['name'])) - #check for duplication and grab old data for defaults + # check for duplication and grab old data for defaults changed = False for field in cfg_fields: old = previous[field] @@ -1986,23 +1986,23 @@ def _grp_req_add(taginfo, grpinfo, reqinfo, block, force, **opts): else: opts[field] = old if block: - #from condition above, either previous is not blocked or force is on, - #either way, we should add the entry + # from condition above, either previous is not blocked or force is on, + # either way, we should add the entry changed = True if not changed: - #no point in adding it again with the same data + # no point in adding it again with the same data return opts.setdefault('type', 'mandatory') opts['group_id'] = group['id'] opts['tag_id'] = tag['id'] opts['req_id'] = req['id'] opts['blocked'] = block - #revoke old entry (if present) + # revoke old entry (if present) update = UpdateProcessor('group_req_listing', values=opts, clauses=['group_id=%(group_id)s', 'tag_id=%(tag_id)s', 'req_id=%(req_id)s']) update.make_revoke() update.execute() - #add new entry + # add new entry insert = InsertProcessor('group_req_listing', data=opts) insert.make_create() insert.execute() @@ -2014,7 +2014,7 @@ def grp_req_remove(taginfo, grpinfo, reqinfo, force=False): Really this shouldn't be used except in special cases Most of the time you really want to use the block or unblock functions """ - #only admins.... + # only admins.... context.session.assertPerm('tag') _grp_req_remove(taginfo, grpinfo, reqinfo, force) @@ -2111,11 +2111,11 @@ def get_tag_groups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True for grp_pkg in _multiRow(q, locals(), fields): grp_id = grp_pkg['group_id'] if grp_id not in groups: - #tag does not have this group + # tag does not have this group continue group = groups[grp_id] if group['blocked']: - #ignore blocked groups + # ignore blocked groups continue pkg_name = grp_pkg['package'] group['packagelist'].setdefault(pkg_name, grp_pkg) @@ -2132,18 +2132,18 @@ def get_tag_groups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True for grp_req in _multiRow(q, locals(), fields): grp_id = grp_req['group_id'] if grp_id not in groups: - #tag does not have this group + # tag does not have this group continue group = groups[grp_id] if group['blocked']: - #ignore blocked groups + # ignore blocked groups continue req_id = grp_req['req_id'] if req_id not in groups: - #tag does not have this group + # tag does not have this group continue elif groups[req_id]['blocked']: - #ignore blocked groups + # ignore blocked groups continue group['grouplist'].setdefault(req_id, grp_req) @@ -2159,7 +2159,7 @@ def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, groups = get_tag_groups(tag, event, inherit, incl_pkgs, incl_reqs) groups = to_list(groups.values()) for group in groups: - #filter blocked entries and collapse to a list + # filter blocked entries and collapse to a list if 'packagelist' in group: if incl_blocked: group['packagelist'] = to_list(group['packagelist'].values()) @@ -2170,7 +2170,7 @@ def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, group['grouplist'] = to_list(group['grouplist'].values()) else: group['grouplist'] = [x for x in group['grouplist'].values() if not x['blocked']] - #filter blocked entries and collapse to a list + # filter blocked entries and collapse to a list if incl_blocked: return groups else: @@ -2267,7 +2267,7 @@ def remove_channel(channel_name, force=False): # check for task references query = QueryProcessor(tables=['task'], clauses=['channel_id=%(channel_id)i'], values=locals(), columns=['id'], opts={'limit':1}) - #XXX slow query + # XXX slow query if query.execute(): raise koji.GenericError('channel %s has task references' % channel_name) query = QueryProcessor(tables=['host_channels'], clauses=['channel_id=%(channel_id)i'], @@ -2316,8 +2316,8 @@ def get_all_arches(): if arches is None: continue for arch in arches.split(): - #in a perfect world, this list would only include canonical - #arches, but not all admins will undertand that. + # in a perfect world, this list would only include canonical + # arches, but not all admins will undertand that. ret[koji.canonArch(arch)] = 1 return to_list(ret.keys()) @@ -2471,7 +2471,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa if event is None: event_id = _singleValue("SELECT get_event()") else: - #make sure event is valid + # make sure event is valid q = "SELECT time FROM events WHERE id=%(event)s" event_time = _singleValue(q, locals(), strict=True) event_id = event @@ -2490,7 +2490,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa repodir = koji.pathinfo.repo(repo_id, tinfo['name']) os.makedirs(repodir) #should not already exist - #generate comps and groups.spec + # generate comps and groups.spec groupsdir = "%s/groups" % (repodir) koji.ensuredir(groupsdir) comps = koji.generate_comps(groups, expand_groups=True) @@ -2510,13 +2510,13 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa with open('%s/repo.json' % repodir, 'w') as fp: json.dump(repo_info, fp, indent=2) - #get build dirs + # get build dirs relpathinfo = koji.PathInfo(topdir='toplink') builddirs = {} for build in builds: relpath = relpathinfo.build(build) builddirs[build['id']] = relpath.lstrip('/') - #generate pkglist files + # generate pkglist files pkglist = {} for repoarch in repo_arches: archdir = joinpath(repodir, repoarch) @@ -2526,7 +2526,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa top_link = joinpath(archdir, 'toplink') os.symlink(top_relpath, top_link) pkglist[repoarch] = open(joinpath(archdir, 'pkglist'), 'w') - #NOTE - rpms is now an iterator + # NOTE - rpms is now an iterator for rpminfo in rpms: if not with_debuginfo and koji.is_debuginfo(rpminfo['name']): continue @@ -2552,7 +2552,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa for repoarch in repo_arches: pkglist[repoarch].close() - #write blocked package lists + # write blocked package lists for repoarch in repo_arches: blocklist = open(joinpath(repodir, repoarch, 'blocklist'), 'w') for pkg in blocks: @@ -2730,7 +2730,7 @@ def repo_delete(repo_id): """Attempt to mark repo deleted, return number of references If the number of references is nonzero, no change is made""" - #get a row lock on the repo + # get a row lock on the repo q = """SELECT state FROM repo WHERE id = %(repo_id)s FOR UPDATE""" _singleValue(q, locals()) references = repo_references(repo_id) @@ -2768,7 +2768,7 @@ def repo_references(repo_id): clauses = ['repo_id=%(repo_id)s', 'retire_event IS NULL'] query = QueryProcessor(columns=fields, aliases=aliases, tables=['standard_buildroot'], clauses=clauses, values=values) - #check results for bad states + # check results for bad states ret = [] for data in query.execute(): if data['state'] == koji.BR_STATES['EXPIRED']: @@ -2813,14 +2813,14 @@ def tag_changed_since_event(event, taglist): Returns: True or False """ data = locals().copy() - #first check the tag_updates table + # first check the tag_updates table clauses = ['update_event > %(event)i', 'tag_id IN %(taglist)s'] query = QueryProcessor(tables=['tag_updates'], columns=['id'], clauses=clauses, values=data, opts={'limit': 1}) if query.execute(): return True - #also check these versioned tables + # also check these versioned tables tables = ( 'tag_listing', 'tag_inheritance', @@ -2890,8 +2890,8 @@ def _create_build_target(name, build_tag, dest_tag): raise koji.GenericError("destination tag '%s' does not exist" % dest_tag) dest_tag = dest_tag_object['id'] - #build targets are versioned, so if the target has previously been deleted, it - #is possible the name is in the system + # build targets are versioned, so if the target has previously been deleted, it + # is possible the name is in the system id = get_build_target_id(name, create=True) insert = InsertProcessor('build_target_config') @@ -3137,7 +3137,7 @@ def _create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_s if not context.opts.get('EnableMaven') and (maven_support or maven_include_all): raise koji.GenericError("Maven support not enabled") - #see if there is already a tag by this name (active) + # see if there is already a tag by this name (active) if get_tag(name): raise koji.GenericError("A tag with the name '%s' already exists" % name) @@ -3150,7 +3150,7 @@ def _create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_s else: parent_id = None - #there may already be an id for a deleted tag, this will reuse it + # there may already be an id for a deleted tag, this will reuse it tag_id = get_tag_id(name, create=True) insert = InsertProcessor('tag_config') @@ -3296,8 +3296,8 @@ def _edit_tag(tagInfo, **kwargs): name = kwargs.get('name') if name and tag['name'] != name: - #attempt to update tag name - #XXX - I'm not sure we should allow this sort of renaming anyway. + # attempt to update tag name + # XXX - I'm not sure we should allow this sort of renaming anyway. # while I can see the convenience, it is an untracked change (granted # a cosmetic one). The more versioning-friendly way would be to create # a new tag with duplicate data and revoke the old tag. This is more @@ -3309,7 +3309,7 @@ def _edit_tag(tagInfo, **kwargs): q = """SELECT id FROM tag WHERE name=%(name)s""" id = _singleValue(q, values, strict=False) if id is not None: - #new name is taken + # new name is taken raise koji.GenericError("Name %s already taken by tag %s" % (name, id)) update = """UPDATE tag SET name = %(name)s @@ -3321,7 +3321,7 @@ WHERE id = %(tagID)i""" if arches and tag['arches'] != arches: kwargs['arches'] = koji.parse_arches(arches, strict=True, allow_none=True) - #check for changes + # check for changes data = tag.copy() changed = False for key in ('perm_id', 'arches', 'locked', 'maven_support', 'maven_include_all'): @@ -3394,7 +3394,7 @@ def delete_tag(tagInfo): def _delete_tag(tagInfo): """Delete the specified tag.""" - #We do not ever DELETE tag data. It is versioned -- we revoke it instead. + # We do not ever DELETE tag data. It is versioned -- we revoke it instead. def _tagDelete(tableName, value, columnName='tag_id'): update = UpdateProcessor(tableName, clauses=["%s = %%(value)i" % columnName], @@ -3406,8 +3406,8 @@ def _delete_tag(tagInfo): tagID = tag['id'] _tagDelete('tag_config', tagID) - #technically, to 'delete' the tag we only have to revoke the tag_config entry - #these remaining revocations are more for cleanup. + # technically, to 'delete' the tag we only have to revoke the tag_config entry + # these remaining revocations are more for cleanup. _tagDelete('tag_extra', tagID) _tagDelete('tag_inheritance', tagID) _tagDelete('tag_inheritance', tagID, 'parent_id') @@ -3890,10 +3890,10 @@ def find_build_id(X, strict=False): AND build.release=%(release)s """ # contraints should ensure this is unique - #log_error(koji.db._quoteparams(q,data)) + # log_error(koji.db._quoteparams(q,data)) c.execute(q, data) r = c.fetchone() - #log_error("%r" % r ) + # log_error("%r" % r ) if not r: if strict: raise koji.GenericError('No matching build found: %r' % X) @@ -4060,7 +4060,7 @@ def _fix_rpm_row(row): row['extra'] = parse_json(row['extra'], desc='rpm extra') return row -#alias for now, may change in the future +# alias for now, may change in the future _fix_archive_row = _fix_rpm_row @@ -4138,7 +4138,7 @@ def get_rpm(rpminfo, strict=False, multi=False): data['external_repo_id'] = get_external_repo_id(data['location'], strict=True) clauses.append("""external_repo_id = %(external_repo_id)i""") elif not multi: - #try to match internal first, otherwise first matching external + # try to match internal first, otherwise first matching external retry = True #if no internal match orig_clauses = list(clauses) #copy clauses.append("""external_repo_id = 0""") @@ -4154,7 +4154,7 @@ def get_rpm(rpminfo, strict=False, multi=False): if ret: return ret if retry: - #at this point we have just an NVRA with no internal match. Open it up to externals + # at this point we have just an NVRA with no internal match. Open it up to externals query.clauses = orig_clauses ret = query.executeOne() if not ret: @@ -4339,7 +4339,7 @@ def get_build_type(buildInfo, strict=False): for (btype,) in query.execute(): ret[btype] = extra.get('typeinfo', {}).get(btype) - #deal with legacy types + # deal with legacy types l_funcs = [['maven', get_maven_build], ['win', get_win_build], ['image', get_image_build]] for ltype, func in l_funcs: @@ -4928,7 +4928,7 @@ def _singleRow(query, values, fields, strict=False): if row: return dict(zip(fields, row)) else: - #strict enforced by _fetchSingle + # strict enforced by _fetchSingle return None def _singleValue(query, values=None, strict=True): @@ -5157,7 +5157,7 @@ def get_buildroot(buildrootID, strict=False): else: return None if len(result) > 1: - #this should be impossible + # this should be impossible raise koji.GenericError("More that one buildroot with id: %i" % buildrootID) return result[0] @@ -5247,7 +5247,7 @@ def _set_build_volume(binfo, volinfo, strict=True): if strict: raise koji.GenericError("Build %(nvr)s already on volume %(volume_name)s" % binfo) else: - #nothing to do + # nothing to do return state = koji.BUILD_STATES[binfo['state']] if state not in ['COMPLETE', 'DELETED']: @@ -5256,7 +5256,7 @@ def _set_build_volume(binfo, volinfo, strict=True): if not os.path.isdir(voldir): raise koji.GenericError("Directory entry missing for volume %(name)s" % volinfo) - #more sanity checks + # more sanity checks for check_vol in list_volumes(): check_binfo = binfo.copy() check_binfo['volume_id'] = check_vol['id'] @@ -5304,7 +5304,7 @@ def _set_build_volume(binfo, volinfo, strict=True): for olddir, newdir in dir_moves: koji.util.rmtree(olddir) - #Fourth, maintain a symlink if appropriate + # Fourth, maintain a symlink if appropriate if volinfo['name'] and volinfo['name'] != 'DEFAULT': base_vol = lookup_name('volume', 'DEFAULT', strict=True) base_binfo = binfo.copy() @@ -5431,7 +5431,7 @@ def new_build(data, strict=False): if 'pkg_id' in data: data['name'] = lookup_package(data['pkg_id'], strict=True)['name'] else: - #see if there's a package name + # see if there's a package name name = data.get('name') if not name: raise koji.GenericError("No name or package id provided for build") @@ -5450,7 +5450,7 @@ def new_build(data, strict=False): else: data['extra'] = None - #provide a few default values + # provide a few default values data.setdefault('state', koji.BUILD_STATES['COMPLETE']) data.setdefault('start_time', 'NOW') data.setdefault('completion_time', 'NOW') @@ -5459,7 +5459,7 @@ def new_build(data, strict=False): data.setdefault('task_id', None) data.setdefault('volume_id', 0) - #check for existing build + # check for existing build old_binfo = get_build(data) if old_binfo: if strict: @@ -5469,7 +5469,7 @@ def new_build(data, strict=False): return old_binfo['id'] koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data) - #insert the new data + # insert the new data insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', 'task_id', 'owner', 'start_time', 'completion_time', 'source', 'extra']) if 'cg_id' in data: @@ -5479,7 +5479,7 @@ def new_build(data, strict=False): insert.execute() new_binfo = get_build(data['id'], strict=True) koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=new_binfo) - #return build_id + # return build_id return data['id'] @@ -5490,7 +5490,7 @@ def recycle_build(old, data): if st_desc == 'BUILDING': # check to see if this is the controlling task if data['state'] == old['state'] and data.get('task_id', '') == old['task_id']: - #the controlling task must have restarted (and called initBuild again) + # the controlling task must have restarted (and called initBuild again) return raise koji.GenericError("Build already in progress (task %(task_id)d)" % old) @@ -5606,7 +5606,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) koji.plugin.run_callbacks('preImport', type='build', srpm=srpm, rpms=rpms, brmap=brmap, task_id=task_id, build_id=build_id, build=None, logs=logs) uploadpath = koji.pathinfo.work() - #verify files exist + # verify files exist for relpath in [srpm] + rpms: fn = "%s/%s" % (uploadpath, relpath) if not os.path.exists(fn): @@ -5614,13 +5614,13 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) rpms = check_noarch_rpms(uploadpath, rpms, logs=logs) - #verify buildroot ids from brmap + # verify buildroot ids from brmap found = {} for br_id in brmap.values(): if br_id in found: continue found[br_id] = 1 - #this will raise an exception if the buildroot id is invalid + # this will raise an exception if the buildroot id is invalid BuildRoot(br_id) # get build informaton @@ -5648,7 +5648,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) binfo = get_build(build_id, strict=True) new_typed_build(binfo, 'rpm') else: - #build_id was passed in - sanity check + # build_id was passed in - sanity check binfo = get_build(build_id, strict=True) st_complete = koji.BUILD_STATES['COMPLETE'] st_old = binfo['state'] @@ -5659,7 +5659,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) if binfo['state'] != koji.BUILD_STATES['BUILDING']: raise koji.GenericError("Unable to complete build: state is %s" \ % koji.BUILD_STATES[binfo['state']]) - #update build state + # update build state update = UpdateProcessor('build', clauses=['id=%(id)s'], values=binfo) update.set(state=st_complete) update.rawset(completion_time='NOW()') @@ -5697,21 +5697,21 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): if not os.path.exists(fn): raise koji.GenericError("no such file: %s" % fn) - #read rpm info + # read rpm info hdr = koji.get_rpm_header(fn) rpminfo = koji.get_header_fields(hdr, ['name', 'version', 'release', 'epoch', 'sourcepackage', 'arch', 'buildtime', 'sourcerpm']) if rpminfo['sourcepackage'] == 1: rpminfo['arch'] = "src" - #sanity check basename + # sanity check basename basename = os.path.basename(fn) expected = "%(name)s-%(version)s-%(release)s.%(arch)s.rpm" % rpminfo if basename != expected: raise koji.GenericError("bad filename: %s (expected %s)" % (basename, expected)) if buildinfo is None: - #figure it out for ourselves + # figure it out for ourselves if rpminfo['sourcepackage'] == 1: buildinfo = get_build(rpminfo, strict=False) if not buildinfo: @@ -5720,10 +5720,10 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): # we add the rpm build type below buildinfo = get_build(build_id, strict=True) else: - #figure it out from sourcerpm string + # figure it out from sourcerpm string buildinfo = get_build(koji.parse_NVRA(rpminfo['sourcerpm'])) if buildinfo is None: - #XXX - handle case where package is not a source rpm + # XXX - handle case where package is not a source rpm # and we still need to create a new build raise koji.GenericError('No matching build') state = koji.BUILD_STATES[buildinfo['state']] @@ -5733,8 +5733,8 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): elif not wrapper: # only enforce the srpm name matching the build for non-wrapper rpms srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % buildinfo - #either the sourcerpm field should match the build, or the filename - #itself (for the srpm) + # either the sourcerpm field should match the build, or the filename + # itself (for the srpm) if rpminfo['sourcepackage'] != 1: if rpminfo['sourcerpm'] != srpmname: raise koji.GenericError("srpm mismatch for %s: %s (expected %s)" \ @@ -5747,7 +5747,7 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): # harmless if build already has this type new_typed_build(buildinfo, 'rpm') - #add rpminfo entry + # add rpminfo entry rpminfo['id'] = _singleValue("""SELECT nextval('rpminfo_id_seq')""") rpminfo['build_id'] = buildinfo['id'] rpminfo['size'] = os.path.getsize(fn) @@ -5773,7 +5773,7 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): koji.plugin.run_callbacks('postImport', type='rpm', rpm=rpminfo, build=buildinfo, filepath=fn, fileinfo=fileinfo) - #extra fields for return + # extra fields for return rpminfo['build'] = buildinfo rpminfo['brootid'] = brootid return rpminfo @@ -5967,7 +5967,7 @@ class CG_Importer(object): raise koji.GenericError("Invalid metadata, cannot encode: %r" % metadata) return metadata if metadata is None: - #default to looking for uploaded file + # default to looking for uploaded file metadata = 'metadata.json' if not isinstance(metadata, six.string_types): raise koji.GenericError("Invalid metadata value: %r" % metadata) @@ -6238,7 +6238,7 @@ class CG_Importer(object): br = BuildRoot() br.cg_new(entry['brinfo']) - #buildroot components + # buildroot components br.setList(entry['rpmlist']) br.updateArchiveList(entry['archives']) @@ -6284,8 +6284,8 @@ class CG_Importer(object): if rinfo['payloadhash'] != comp['sigmd5']: # XXX - this is a temporary workaround until we can better track external refs logger.warning("IGNORING rpm component (md5 mismatch): %r", comp) - #nvr = "%(name)s-%(version)s-%(release)s" % rinfo - #raise koji.GenericError("md5sum mismatch for %s: %s != %s" + # nvr = "%(name)s-%(version)s-%(release)s" % rinfo + # raise koji.GenericError("md5sum mismatch for %s: %s != %s" # % (nvr, comp['sigmd5'], rinfo['payloadhash'])) # TODO - should we check the signature field? return rinfo @@ -6305,7 +6305,7 @@ class CG_Importer(object): continue if archive['checksum'] == comp['checksum']: return archive - #else + # else logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) if type_mismatches: logger.error("Match failed with %i type mismatches", type_mismatches) @@ -6313,7 +6313,7 @@ class CG_Importer(object): # XXX - this is a temporary workaround until we can better track external refs logger.warning("IGNORING unmatched archive: %r", comp) return None - #raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + # raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) def match_kojifile(self, comp): """Look up the file by archive id and sanity check the other data""" @@ -6520,7 +6520,7 @@ def add_external_rpm(rpminfo, external_repo, strict=True): if field not in rpminfo: raise koji.GenericError("%s field missing: %r" % (field, rpminfo)) if not isinstance(rpminfo[field], allowed): - #this will catch unwanted NULLs + # this will catch unwanted NULLs raise koji.GenericError("Invalid value for %s: %r" % (field, rpminfo[field])) # strip extra fields rpminfo = dslice(rpminfo, [x[0] for x in dtypes]) @@ -6762,7 +6762,7 @@ def get_archive_type(filename=None, type_name=None, type_id=None, strict=False): elif len(results) > 1: # this should never happen, and is a misconfiguration in the database raise koji.GenericError('multiple matches for file extension: %s' % ext) - #otherwise + # otherwise if strict: raise koji.GenericError('unsupported file extension: %s' % ext) else: @@ -7085,7 +7085,7 @@ def _generate_maven_metadata(mavendir): def add_rpm_sig(an_rpm, sighdr): """Store a signature header for an rpm""" - #calling function should perform permission checks, if applicable + # calling function should perform permission checks, if applicable rinfo = get_rpm(an_rpm, strict=True) if rinfo['external_repo_id']: raise koji.GenericError("Not an internal rpm: %s (from %s)" \ @@ -7116,7 +7116,7 @@ def add_rpm_sig(an_rpm, sighdr): raise koji.GenericError("wrong md5 for %s: %s" % (nvra, sigmd5)) if not sigkey: sigkey = '' - #we use the sigkey='' to represent unsigned in the db (so that uniqueness works) + # we use the sigkey='' to represent unsigned in the db (so that uniqueness works) else: sigkey = koji.get_sigpacket_key_id(sigkey) sighash = hashlib.md5(sighdr).hexdigest() @@ -7125,7 +7125,7 @@ def add_rpm_sig(an_rpm, sighdr): q = """SELECT sighash FROM rpmsigs WHERE rpm_id=%(rpm_id)i AND sigkey=%(sigkey)s""" rows = _fetchMulti(q, locals()) if rows: - #TODO[?] - if sighash is the same, handle more gracefully + # TODO[?] - if sighash is the same, handle more gracefully nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rinfo raise koji.GenericError("Signature already exists for package %s, key %s" % (nvra, sigkey)) koji.plugin.run_callbacks('preRPMSign', sigkey=sigkey, sighash=sighash, build=binfo, rpm=rinfo) @@ -7146,24 +7146,24 @@ def _scan_sighdr(sighdr, fn): raise koji.GenericError("No such path: %s" % fn) if not os.path.isfile(fn): raise koji.GenericError("Not a regular file: %s" % fn) - #XXX should probably add an option to splice_rpm_sighdr to handle this instead + # XXX should probably add an option to splice_rpm_sighdr to handle this instead sig_start, sigsize = koji.find_rpm_sighdr(fn) hdr_start = sig_start + sigsize hdrsize = koji.rpm_hdr_size(fn, hdr_start) inp = open(fn, 'rb') outp = tempfile.TemporaryFile(mode='w+b') - #before signature + # before signature outp.write(inp.read(sig_start)) - #signature + # signature outp.write(sighdr) inp.seek(sigsize, 1) - #main header + # main header outp.write(inp.read(hdrsize)) inp.close() outp.seek(0, 0) ts = rpm.TransactionSet() ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS) - #(we have no payload, so verifies would fail otherwise) + # (we have no payload, so verifies would fail otherwise) hdr = ts.hdrFromFdno(outp.fileno()) outp.close() sig = koji.get_header_field(hdr, 'siggpg') @@ -7172,7 +7172,7 @@ def _scan_sighdr(sighdr, fn): return koji.get_header_field(hdr, 'sigmd5'), sig def check_rpm_sig(an_rpm, sigkey, sighdr): - #verify that the provided signature header matches the key and rpm + # verify that the provided signature header matches the key and rpm rinfo = get_rpm(an_rpm, strict=True) binfo = get_build(rinfo['build_id']) builddir = koji.pathinfo.build(binfo) @@ -7234,7 +7234,7 @@ def write_signed_rpm(an_rpm, sigkey, force=False): raise koji.GenericError("No such path: %s" % rpm_path) if not os.path.isfile(rpm_path): raise koji.GenericError("Not a regular file: %s" % rpm_path) - #make sure we have it in the db + # make sure we have it in the db rpm_id = rinfo['id'] q = """SELECT sighash FROM rpmsigs WHERE rpm_id=%(rpm_id)i AND sigkey=%(sigkey)s""" row = _fetchSingle(q, locals()) @@ -7244,7 +7244,7 @@ def write_signed_rpm(an_rpm, sigkey, force=False): signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rinfo, sigkey)) if os.path.exists(signedpath): if not force: - #already present + # already present return else: os.unlink(signedpath) @@ -7289,7 +7289,7 @@ def query_history(tables=None, **kwargs): cg: only relating to a content generator """ common_fields = { - #fields:aliases common to all versioned tables + # fields:aliases common to all versioned tables 'active': 'active', 'create_event': 'create_event', 'revoke_event': 'revoke_event', @@ -7329,12 +7329,12 @@ def query_history(tables=None, **kwargs): 'group_package_listing': ['group_id', 'tag_id', 'package', 'blocked', 'type', 'basearchonly', 'requires'], } name_joins = { - #joins triggered by table fields for name lookup - #field : [table, join-alias, alias] + # joins triggered by table fields for name lookup + # field : [table, join-alias, alias] 'user_id': ['users', 'users', 'user'], 'perm_id': ['permissions', 'permission'], 'cg_id': ['content_generator'], - #group_id is overloaded (special case below) + # group_id is overloaded (special case below) 'tag_id': ['tag'], 'host_id': ['host'], 'channel_id': ['channels'], @@ -7374,7 +7374,7 @@ def query_history(tables=None, **kwargs): joined[tbl] = join_as fullname = "%s.name" % join_as if len(name_join) > 2: - #apply alias + # apply alias fields[fullname] = "%s.name" % name_join[2] else: fields[fullname] = fullname @@ -7383,7 +7383,7 @@ def query_history(tables=None, **kwargs): else: joins.append('LEFT OUTER JOIN %s AS %s ON %s = %s.id' % (tbl, join_as, field, join_as)) elif field == 'build_id': - #special case + # special case fields.update({ 'package.name': 'name', #XXX? 'build.version': 'version', @@ -7417,7 +7417,7 @@ def query_history(tables=None, **kwargs): break data['tag_id'] = get_tag_id(value, strict=True) if table == 'tag_inheritance': - #special cased because there are two tag columns + # special cased because there are two tag columns clauses.append("tag_id = %(tag_id)i OR parent_id = %(tag_id)i") else: clauses.append("%s.id = %%(tag_id)i" % joined['tag']) @@ -7504,7 +7504,7 @@ def query_history(tables=None, **kwargs): clauses.append('ev1.time > %(after)s OR ev2.time > %(after)s') fields['ev1.time > %(after)s'] = '_created_after' fields['ev2.time > %(after)s'] = '_revoked_after' - #clauses.append('EXTRACT(EPOCH FROM ev1.time) > %(after)s OR EXTRACT(EPOCH FROM ev2.time) > %(after)s') + # clauses.append('EXTRACT(EPOCH FROM ev1.time) > %(after)s OR EXTRACT(EPOCH FROM ev2.time) > %(after)s') elif arg == 'afterEvent': data['afterEvent'] = value c_test = '%s.create_event > %%(afterEvent)i' % table @@ -7517,7 +7517,7 @@ def query_history(tables=None, **kwargs): value = datetime.datetime.fromtimestamp(value).isoformat(' ') data['before'] = value clauses.append('ev1.time < %(before)s OR ev2.time < %(before)s') - #clauses.append('EXTRACT(EPOCH FROM ev1.time) < %(before)s OR EXTRACT(EPOCH FROM ev2.time) < %(before)s') + # clauses.append('EXTRACT(EPOCH FROM ev1.time) < %(before)s OR EXTRACT(EPOCH FROM ev2.time) < %(before)s') fields['ev1.time < %(before)s'] = '_created_before' fields['ev2.time < %(before)s'] = '_revoked_before' elif arg == 'beforeEvent': @@ -7605,13 +7605,13 @@ def untagged_builds(name=None, queryOpts=None): joins.append("""LEFT OUTER JOIN tag_listing ON tag_listing.build_id = build.id AND tag_listing.active = TRUE""") clauses = ["tag_listing.tag_id IS NULL", "build.state = %(st_complete)i"] - #q = """SELECT build.id, package.name, build.version, build.release - #FROM build + # q = """SELECT build.id, package.name, build.version, build.release + # FROM build # JOIN package on package.id = build.pkg_id # LEFT OUTER JOIN tag_listing ON tag_listing.build_id = build.id # AND tag_listing.active IS TRUE - #WHERE tag_listing.tag_id IS NULL AND build.state = %(st_complete)i""" - #return _multiRow(q, locals(), aliases) + # WHERE tag_listing.tag_id IS NULL AND build.state = %(st_complete)i""" + # return _multiRow(q, locals(), aliases) query = QueryProcessor(columns=fields, aliases=aliases, tables=tables, joins=joins, clauses=clauses, values=locals(), opts=queryOpts) @@ -7640,7 +7640,7 @@ def build_references(build_id, limit=None, lazy=False): if lazy and ret['tags']: return ret - #we'll need the component rpm and archive ids for the rest + # we'll need the component rpm and archive ids for the rest q = """SELECT id FROM rpminfo WHERE build_id=%(build_id)i""" build_rpm_ids = _fetchMulti(q, locals()) q = """SELECT id FROM archiveinfo WHERE build_id=%(build_id)i""" @@ -7869,7 +7869,7 @@ def reset_build(build): context.session.assertPerm('admin') binfo = get_build(build) if not binfo: - #nothing to do + # nothing to do return st_old = binfo['state'] koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=koji.BUILD_STATES['CANCELED'], info=binfo) @@ -8045,7 +8045,7 @@ def get_notification_recipients(build, tag_id, state): 'user_id': owner['id'], 'email': '%s@%s' % (owner['name'], email_domain) }) - #FIXME - if tag_id is None, we don't have a good way to get the package owner. + # FIXME - if tag_id is None, we don't have a good way to get the package owner. # using all package owners from all tags would be way overkill. if not recipients: @@ -8156,7 +8156,7 @@ def add_group_member(group, user, strict=True): raise koji.GenericError("Not an user: %s" % user) if uinfo['usertype'] == koji.USERTYPES['GROUP']: raise koji.GenericError("Groups cannot be members of other groups") - #check to see if user is already a member + # check to see if user is already a member data = {'user_id' : uinfo['id'], 'group_id' : ginfo['id']} table = 'user_groups' clauses = ('user_id = %(user_id)i', 'group_id = %(group_id)s') @@ -8945,8 +8945,8 @@ def policy_get_pkg(data): if 'package' in data: pkginfo = lookup_package(data['package'], strict=False) if not pkginfo: - #for some operations (e.g. adding a new package), the package - #entry may not exist yet + # for some operations (e.g. adding a new package), the package + # entry may not exist yet if isinstance(data['package'], six.string_types): return {'id' : None, 'name' : data['package']} else: @@ -8955,7 +8955,7 @@ def policy_get_pkg(data): if 'build' in data: binfo = get_build(data['build'], strict=True) return {'id' : binfo['package_id'], 'name' : binfo['name']} - #else + # else raise koji.GenericError("policy requires package data") @@ -8968,7 +8968,7 @@ def policy_get_version(data): return data['version'] if 'build' in data: return get_build(data['build'], strict=True)['version'] - #else + # else raise koji.GenericError("policy requires version data") @@ -8981,7 +8981,7 @@ def policy_get_release(data): return data['release'] if 'build' in data: return get_build(data['build'], strict=True)['release'] - #else + # else raise koji.GenericError("policy requires release data") @@ -9053,7 +9053,7 @@ class PackageTest(koji.policy.MatchTest): name = 'package' field = '_package' def run(self, data): - #we need to find the package name from the base data + # we need to find the package name from the base data data[self.field] = policy_get_pkg(data)['name'] return super(PackageTest, self).run(data) @@ -9072,7 +9072,7 @@ class ReleaseTest(koji.policy.MatchTest): name = 'release' field = '_release' def run(self, data): - #we need to find the build NVR from the base data + # we need to find the build NVR from the base data data[self.field] = policy_get_release(data) return super(ReleaseTest, self).run(data) @@ -9082,7 +9082,7 @@ class VolumeTest(koji.policy.MatchTest): name = 'volume' field = '_volume' def run(self, data): - #we need to find the volume name from the base data + # we need to find the volume name from the base data volinfo = None if 'volume' in data: volinfo = lookup_name('volume', data['volume'], strict=False) @@ -9105,7 +9105,7 @@ class CGMatchAnyTest(koji.policy.BaseSimpleTest): name = 'cg_match_any' def run(self, data): - #we need to find the volume name from the base data + # we need to find the volume name from the base data cgs = policy_get_cgs(data) patterns = self.str.split()[1:] for cg_name in cgs: @@ -9128,7 +9128,7 @@ class CGMatchAllTest(koji.policy.BaseSimpleTest): name = 'cg_match_all' def run(self, data): - #we need to find the volume name from the base data + # we need to find the volume name from the base data cgs = policy_get_cgs(data) if not cgs: return False @@ -9157,7 +9157,7 @@ class TagTest(koji.policy.MatchTest): return get_tag(tag, strict=False) def run(self, data): - #we need to find the tag name from the base data + # we need to find the tag name from the base data tinfo = self.get_tag(data) if tinfo is None: return False @@ -9179,13 +9179,13 @@ class HasTagTest(koji.policy.BaseSimpleTest): if 'build' not in data: return False tags = list_tags(build=data['build']) - #True if any of these tags match any of the patterns + # True if any of these tags match any of the patterns args = self.str.split()[1:] for tag in tags: for pattern in args: if fnmatch.fnmatch(tag['name'], pattern): return True - #otherwise... + # otherwise... return False class SkipTagTest(koji.policy.BaseSimpleTest): @@ -9213,7 +9213,7 @@ class BuildTagTest(koji.policy.BaseSimpleTest): continue if multi_fnmatch(tagname, args): return True - #otherwise... + # otherwise... return False @@ -9289,7 +9289,7 @@ class IsBuildOwnerTest(koji.policy.BaseSimpleTest): # owner is a group, check to see if user is a member if owner['id'] in koji.auth.get_user_groups(user['id']): return True - #otherwise... + # otherwise... return False class UserInGroupTest(koji.policy.BaseSimpleTest): @@ -9309,7 +9309,7 @@ class UserInGroupTest(koji.policy.BaseSimpleTest): for pattern in args: if fnmatch.fnmatch(group, pattern): return True - #otherwise... + # otherwise... return False class HasPermTest(koji.policy.BaseSimpleTest): @@ -9329,7 +9329,7 @@ class HasPermTest(koji.policy.BaseSimpleTest): for pattern in args: if fnmatch.fnmatch(perm, pattern): return True - #otherwise... + # otherwise... return False class SourceTest(koji.policy.MatchTest): @@ -9351,11 +9351,11 @@ class SourceTest(koji.policy.MatchTest): # no source to match against return False else: - #crack open the build task + # crack open the build task task = Task(build['task_id']) info = task.getInfo() params = task.getRequest() - #signatures: + # signatures: # build - (src, target, opts=None) # maven - (url, target, opts=None) # winbuild - (name, source_url, target, opts=None) @@ -9391,7 +9391,7 @@ class PolicyTest(koji.policy.BaseSimpleTest): def run(self, data): args = self.str.split()[1:] if self.depth != 0: - #LOOP! + # LOOP! raise koji.GenericError("encountered policy loop at %s" % self.str) ruleset = context.policy.get(args[0]) if not ruleset: @@ -9914,18 +9914,18 @@ class RootExports(object): return _singleRow(q, values, fields, strict=True) def makeTask(self, *args, **opts): - #this is mainly for debugging - #only an admin can make arbitrary tasks + # this is mainly for debugging + # only an admin can make arbitrary tasks context.session.assertPerm('admin') return make_task(*args, **opts) def uploadFile(self, path, name, size, md5sum, offset, data, volume=None): - #path: the relative path to upload to - #name: the name of the file - #size: size of contents (bytes) - #md5: md5sum (hex digest) of contents - #data: base64 encoded file contents - #offset: the offset of the chunk + # path: the relative path to upload to + # name: the name of the file + # size: size of contents (bytes) + # md5: md5sum (hex digest) of contents + # data: base64 encoded file contents + # offset: the offset of the chunk # files can be uploaded in chunks, if so the md5 and size describe # the chunk rather than the whole file. the offset indicates where # the chunk belongs @@ -9963,7 +9963,7 @@ class RootExports(object): if not stat.S_ISREG(st.st_mode): raise koji.GenericError("destination not a file: %s" % fn) elif offset == 0: - #first chunk, so file should not exist yet + # first chunk, so file should not exist yet if not fn.endswith('.log'): # but we allow .log files to be uploaded multiple times to support # realtime log-file viewing @@ -9972,7 +9972,7 @@ class RootExports(object): # log_error("fd=%r" %fd) try: if offset == 0 or (offset == -1 and size == len(contents)): - #truncate file + # truncate file fcntl.lockf(fd, fcntl.LOCK_EX|fcntl.LOCK_NB) try: os.ftruncate(fd, 0) @@ -9983,7 +9983,7 @@ class RootExports(object): os.lseek(fd, 0, 2) else: os.lseek(fd, offset, 0) - #write contents + # write contents fcntl.lockf(fd, fcntl.LOCK_EX|fcntl.LOCK_NB, len(contents), 0, 2) try: os.write(fd, contents) @@ -9992,7 +9992,7 @@ class RootExports(object): fcntl.lockf(fd, fcntl.LOCK_UN, len(contents), 0, 2) if offset == -1: if size is not None: - #truncate file + # truncate file fcntl.lockf(fd, fcntl.LOCK_EX|fcntl.LOCK_NB) try: os.ftruncate(fd, size) @@ -10000,7 +10000,7 @@ class RootExports(object): finally: fcntl.lockf(fd, fcntl.LOCK_UN) if verify is not None: - #check final digest + # check final digest chksum = sum_cls() fcntl.lockf(fd, fcntl.LOCK_SH|fcntl.LOCK_NB) try: @@ -10323,7 +10323,7 @@ class RootExports(object): The return value is the task id """ context.session.assertLogin() - #first some lookups and basic sanity checks + # first some lookups and basic sanity checks build = get_build(build, strict=True) tag = get_tag(tag, strict=True) if fromtag: @@ -10359,11 +10359,11 @@ class RootExports(object): policy_data['operation'] = 'tag' else: policy_data['operation'] = 'move' - #don't check policy for admins using force + # don't check policy for admins using force if not (force and context.session.hasPerm('admin')): assert_policy('tag', policy_data) - #XXX - we're running this check twice, here and in host.tagBuild (called by the task) - #spawn the tagging task + # XXX - we're running this check twice, here and in host.tagBuild (called by the task) + # spawn the tagging task return make_task('tagBuild', [tag_id, build_id, force, fromtag_id], priority=10) def untagBuild(self, tag, build, strict=True, force=False): @@ -10371,7 +10371,7 @@ class RootExports(object): Unlike tagBuild, this does not create a task No return value""" - #we can't staticmethod this one -- we're limiting the options + # we can't staticmethod this one -- we're limiting the options context.session.assertLogin() user_id = context.session.user_id tag_id = get_tag(tag, strict=True)['id'] @@ -10379,7 +10379,7 @@ class RootExports(object): policy_data = {'tag' : None, 'build' : build_id, 'fromtag' : tag_id} policy_data['operation'] = 'untag' try: - #don't check policy for admins using force + # don't check policy for admins using force if not (force and context.session.hasPerm('admin')): assert_policy('tag', policy_data) _untag_build(tag, build, strict=strict, force=force) @@ -10420,7 +10420,7 @@ class RootExports(object): Returns the task id of the task performing the move""" context.session.assertLogin() - #lookups and basic sanity checks + # lookups and basic sanity checks pkg_id = get_package_id(package, strict=True) tag1_id = get_tag_id(tag1, strict=True) tag2_id = get_tag_id(tag2, strict=True) @@ -10440,7 +10440,7 @@ class RootExports(object): else: raise koji.TagError(pkg_error) - #access check + # access check assert_tag_access(tag1_id, user_id=None, force=force) assert_tag_access(tag2_id, user_id=None, force=force) @@ -10448,14 +10448,14 @@ class RootExports(object): # we want 'ORDER BY tag_listing.create_event ASC' not DESC so reverse build_list.reverse() - #policy check + # policy check policy_data = {'tag' : tag2, 'fromtag' : tag1, 'operation' : 'move'} - #don't check policy for admins using force + # don't check policy for admins using force if not (force and context.session.hasPerm('admin')): for build in build_list: policy_data['build'] = build['id'] assert_policy('tag', policy_data) - #XXX - we're running this check twice, here and in host.tagBuild (called by the task) + # XXX - we're running this check twice, here and in host.tagBuild (called by the task) wait_on = [] tasklist = [] @@ -10629,13 +10629,13 @@ class RootExports(object): if not task.verifyOwner() and not task.verifyHost(): if not context.session.hasPerm('admin'): raise koji.ActionNotAllowed('Cannot cancel task, not owner') - #non-admins can also use cancelBuild + # non-admins can also use cancelBuild task.cancel(recurse=recurse) def cancelTaskFull(self, task_id, strict=True): """Cancel a task and all tasks in its group""" context.session.assertPerm('admin') - #non-admins can use cancelBuild or cancelTask + # non-admins can use cancelBuild or cancelTask Task(task_id).cancelFull(strict=strict) def cancelTaskChildren(self, task_id): @@ -10657,7 +10657,7 @@ class RootExports(object): def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, owner=None, type=None): """List builds tagged with tag""" - #lookup tag id + # lookup tag id tag = get_tag(tag, strict=True, event=event)['id'] results = readTaggedBuilds(tag, event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) if prefix: @@ -10667,7 +10667,7 @@ class RootExports(object): def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, rpmsigs=False, owner=None, type=None): """List rpms and builds within tag""" - #lookup tag id + # lookup tag id tag = get_tag(tag, strict=True, event=event)['id'] return readTaggedRPMS(tag, event=event, inherit=inherit, latest=latest, package=package, arch=arch, rpmsigs=rpmsigs, owner=owner, type=type) @@ -10853,14 +10853,14 @@ class RootExports(object): def getLatestBuilds(self, tag, event=None, package=None, type=None): """List latest builds for tag (inheritance enabled)""" if not isinstance(tag, six.integer_types): - #lookup tag id + # lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type) def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None): """List latest RPMS for tag (inheritance enabled)""" if not isinstance(tag, six.integer_types): - #lookup tag id + # lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, latest=True, rpmsigs=rpmsigs, type=type) @@ -10950,7 +10950,7 @@ class RootExports(object): if jumps is None: jumps = {} if not isinstance(tag, six.integer_types): - #lookup tag id + # lookup tag id tag = get_tag_id(tag, strict=True) for mapping in [stops, jumps]: for key in to_list(mapping.keys()): @@ -10977,7 +10977,7 @@ class RootExports(object): If no build has the given ID, or the build generated no RPMs, an empty list is returned.""" if not isinstance(build, six.integer_types): - #lookup build id + # lookup build id build = self.findBuildID(build, strict=True) return self.listRPMs(buildID=build) @@ -11170,7 +11170,7 @@ class RootExports(object): def writeSignedRPM(self, an_rpm, sigkey, force=False): """Write a signed copy of the rpm""" context.session.assertPerm('sign') - #XXX - still not sure if this is the right restriction + # XXX - still not sure if this is the right restriction return write_signed_rpm(an_rpm, sigkey, force) def addRPMSig(self, an_rpm, data): @@ -11287,7 +11287,7 @@ class RootExports(object): if pkg_id not in pkgs: return False else: - #still might be blocked + # still might be blocked return not pkgs[pkg_id]['blocked'] def getPackageConfig(self, tag, pkg, event=None): @@ -11380,7 +11380,7 @@ class RootExports(object): grantCGAccess = staticmethod(grant_cg_access) revokeCGAccess = staticmethod(revoke_cg_access) - #group management calls + # group management calls newGroup = staticmethod(new_group) addGroupMember = staticmethod(add_group_member) dropGroupMember = staticmethod(drop_group_member) @@ -11423,7 +11423,7 @@ class RootExports(object): """Return build configuration associated with a tag""" taginfo = get_tag(tag, strict=True, event=event) order = readFullInheritance(taginfo['id'], event=event) - #follow inheritance for arches and extra + # follow inheritance for arches and extra for link in order: if link['noconfig']: continue @@ -11742,7 +11742,7 @@ class RootExports(object): if val: try: if val.find('//task_ scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] @@ -13168,10 +13168,10 @@ class HostExports(object): """ host = Host() host.verify() - #sanity checks + # sanity checks task = Task(data['task_id']) task.assertHost(host.id) - #prep the data + # prep the data data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] data['completion_time'] = None @@ -13182,7 +13182,7 @@ class HostExports(object): def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None): """Import final build contents into the database""" - #sanity checks + # sanity checks host = Host() host.verify() task = Task(task_id) @@ -13424,7 +13424,7 @@ class HostExports(object): raise koji.GenericError('Windows support not enabled') host = Host() host.verify() - #sanity checks + # sanity checks task = Task(task_id) task.assertHost(host.id) # build_info must contain name, version, and release @@ -13863,22 +13863,22 @@ class HostExports(object): repo_expire(repo_id) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) return - #else: + # else: repo_ready(repo_id) repo_expire_older(rinfo['tag_id'], rinfo['create_event'], rinfo['dist']) - #make a latest link + # make a latest link if rinfo['dist']: latestrepolink = koji.pathinfo.distrepo('latest', rinfo['tag_name']) else: latestrepolink = koji.pathinfo.repo('latest', rinfo['tag_name']) - #XXX - this is a slight abuse of pathinfo + # XXX - this is a slight abuse of pathinfo try: if os.path.lexists(latestrepolink): os.unlink(latestrepolink) os.symlink(str(repo_id), latestrepolink) except OSError: - #making this link is nonessential + # making this link is nonessential log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) @@ -14061,7 +14061,7 @@ def handle_upload(environ): if not context.session.logged_in: raise koji.ActionNotAllowed('you must be logged-in to upload a file') args = parse_qs(environ.get('QUERY_STRING', ''), strict_parsing=True) - #XXX - already parsed by auth + # XXX - already parsed by auth name = args['filename'][0] path = args.get('filepath', ('',))[0] verify = args.get('fileverify', ('',))[0] diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 8515a1a..28ebee4 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -64,7 +64,7 @@ class HandlerRegistry(object): def __init__(self): self.funcs = {} - #introspection functions + # introspection functions self.register_function(self.list_api, name="_listapi") self.register_function(self.system_listMethods, name="system.listMethods") self.register_function(self.system_methodSignature, name="system.methodSignature") @@ -106,7 +106,7 @@ class HandlerRegistry(object): """ for v in six.itervalues(vars(plugin)): if isinstance(v, type): - #skip classes + # skip classes continue if callable(v): if getattr(v, 'exported', False): @@ -138,8 +138,8 @@ class HandlerRegistry(object): def list_api(self): funcs = [] for name, func in self.funcs.items(): - #the keys in self.funcs determine the name of the method as seen over xmlrpc - #func.__name__ might differ (e.g. for dotted method names) + # the keys in self.funcs determine the name of the method as seen over xmlrpc + # func.__name__ might differ (e.g. for dotted method names) args = self._getFuncArgs(func) argspec = self.getargspec(func) funcs.append({'name': name, @@ -164,7 +164,7 @@ class HandlerRegistry(object): return koji.util.to_list(self.funcs.keys()) def system_methodSignature(self, method): - #it is not possible to autogenerate this data + # it is not possible to autogenerate this data return 'signatures not supported' def system_methodHelp(self, method): @@ -268,7 +268,7 @@ class ModXMLRPCRequestHandler(object): return response def handle_upload(self, environ): - #uploads can't be in a multicall + # uploads can't be in a multicall context.method = None self.check_session() self.enforce_lockout() @@ -280,13 +280,13 @@ class ModXMLRPCRequestHandler(object): def check_session(self): if not hasattr(context, "session"): - #we may be called again by one of our meta-calls (like multiCall) - #so we should only create a session if one does not already exist + # we may be called again by one of our meta-calls (like multiCall) + # so we should only create a session if one does not already exist context.session = koji.auth.Session() try: context.session.validate() except koji.AuthLockError: - #might be ok, depending on method + # might be ok, depending on method if context.method not in ('exclusiveSession', 'login', 'krbLogin', 'logout'): raise @@ -359,7 +359,7 @@ class ModXMLRPCRequestHandler(object): """Handle a single XML-RPC request""" pass - #XXX no longer used + # XXX no longer used def offline_reply(start_response, msg=None): @@ -395,13 +395,13 @@ def load_config(environ): - all PythonOptions (except ConfigFile) are now deprecated and support for them will disappear in a future version of Koji """ - #get our config file(s) + # get our config file(s) cf = environ.get('koji.hub.ConfigFile', '/etc/koji-hub/hub.conf') cfdir = environ.get('koji.hub.ConfigDir', '/etc/koji-hub/hub.conf.d') config = koji.read_config_files([cfdir, (cf, True)], raw=True) cfgmap = [ - #option, type, default + # option, type, default ['DBName', 'string', None], ['DBUser', 'string', None], ['DBHost', 'string', None], @@ -479,7 +479,7 @@ def load_config(environ): # load policies # (only from config file) if config and config.has_section('policy'): - #for the moment, we simply transfer the policy conf to opts + # for the moment, we simply transfer the policy conf to opts opts['policy'] = dict(config.items('policy')) else: opts['policy'] = {} @@ -504,7 +504,7 @@ def load_plugins(opts): tracker.load(name) except Exception: logger.error(''.join(traceback.format_exception(*sys.exc_info()))) - #make this non-fatal, but set ServerOffline + # make this non-fatal, but set ServerOffline opts['ServerOffline'] = True opts['OfflineMessage'] = 'configuration error' return tracker @@ -542,7 +542,7 @@ _default_policies = { def get_policy(opts, plugins): if not opts.get('policy'): return - #first find available policy tests + # first find available policy tests alltests = [koji.policy.findSimpleTests([vars(kojihub), vars(koji.policy)])] # we delay merging these to allow a test to be overridden for a specific policy for plugin_name in opts.get('Plugins', '').split(): @@ -552,7 +552,7 @@ def get_policy(opts, plugins): alltests.append(koji.policy.findSimpleTests(vars(plugin))) policy = {} for pname, text in six.iteritems(opts['policy']): - #filter/merge tests + # filter/merge tests merged = {} for tests in alltests: # tests can be limited to certain policies by setting a class variable @@ -598,7 +598,7 @@ def setup_logging1(): global log_handler logger = logging.getLogger("koji") logger.setLevel(logging.WARNING) - #stderr logging (stderr goes to httpd logs) + # stderr logging (stderr goes to httpd logs) log_handler = logging.StreamHandler() log_format = '%(asctime)s [%(levelname)s] SETUP p=%(process)s %(name)s: %(message)s' log_handler.setFormatter(HubFormatter(log_format)) @@ -608,7 +608,7 @@ def setup_logging1(): def setup_logging2(opts): global log_handler """Adjust logging based on configuration options""" - #determine log level + # determine log level level = opts['LogLevel'] valid_levels = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL') # the config value can be a single level name or a series of @@ -624,7 +624,7 @@ def setup_logging2(opts): default = level if level not in valid_levels: raise koji.GenericError("Invalid log level: %s" % level) - #all our loggers start with koji + # all our loggers start with koji if name == '': name = 'koji' default = level @@ -639,9 +639,9 @@ def setup_logging2(opts): if opts.get('KojiDebug'): logger.setLevel(logging.DEBUG) elif default is None: - #LogLevel did not configure a default level + # LogLevel did not configure a default level logger.setLevel(logging.WARNING) - #log_handler defined in setup_logging1 + # log_handler defined in setup_logging1 log_handler.setFormatter(HubFormatter(opts['LogFormat'])) @@ -746,7 +746,7 @@ def application(environ, start_response): ] start_response('200 OK', headers) if h.traceback: - #rollback + # rollback context.cnx.rollback() elif context.commit_pending: # Currently there is not much data we can provide to the @@ -764,7 +764,7 @@ def application(environ, start_response): h.logger.debug("Returning %d bytes after %f seconds", len(response), time.time() - start) finally: - #make sure context gets cleaned up + # make sure context gets cleaned up if hasattr(context, 'cnx'): try: context.cnx.close() diff --git a/koji/__init__.py b/koji/__init__.py index 16720f5..8c3a677 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -128,7 +128,7 @@ for h in ( 'RECOMMENDNAME', 'RECOMMENDVERSION', 'RECOMMENDFLAGS'): SUPPORTED_OPT_DEP_HDRS[h] = hasattr(rpm, 'RPMTAG_%s' % h) -## BEGIN kojikamid dup +# BEGIN kojikamid dup # class Enum(dict): """A simple class to track our enumerated constants @@ -167,7 +167,7 @@ class Enum(dict): # deprecated getvalue = _notImplemented - #read-only + # read-only __setitem__ = _notImplemented __delitem__ = _notImplemented clear = _notImplemented @@ -176,7 +176,7 @@ class Enum(dict): update = _notImplemented setdefault = _notImplemented -## END kojikamid dup +# END kojikamid dup # API_VERSION = 1 @@ -215,7 +215,7 @@ AUTHTYPE_KERB = 1 AUTHTYPE_SSL = 2 AUTHTYPE_GSSAPI = 3 -#dependency types +# dependency types DEP_REQUIRE = 0 DEP_PROVIDE = 1 DEP_OBSOLETE = 2 @@ -225,7 +225,7 @@ DEP_ENHANCE = 5 DEP_SUPPLEMENT = 6 DEP_RECOMMEND = 7 -#dependency flags +# dependency flags RPMSENSE_LESS = 2 RPMSENSE_GREATER = 4 RPMSENSE_EQUAL = 8 @@ -266,7 +266,7 @@ TAG_UPDATE_TYPES = Enum(( 'MANUAL', )) -## BEGIN kojikamid dup +# BEGIN kojikamid dup # CHECKSUM_TYPES = Enum(( 'md5', @@ -274,9 +274,9 @@ CHECKSUM_TYPES = Enum(( 'sha256', )) -## END kojikamid dup +# END kojikamid dup # -#PARAMETERS +# PARAMETERS BASEDIR = '/mnt/koji' # default task priority PRIO_DEFAULT = 20 @@ -285,9 +285,9 @@ PRIO_DEFAULT = 20 DEFAULT_REQUEST_TIMEOUT = 60 * 60 * 12 DEFAULT_AUTH_TIMEOUT = 60 -## BEGIN kojikamid dup +# BEGIN kojikamid dup # -#Exceptions +# Exceptions PythonImportError = ImportError # will be masked by koji's one class GenericError(Exception): @@ -302,7 +302,7 @@ class GenericError(Exception): return str(self.args[0]) except: return str(self.__dict__) -## END kojikamid dup +# END kojikamid dup # class LockError(GenericError): """Raised when there is a lock conflict""" @@ -320,12 +320,12 @@ class ActionNotAllowed(GenericError): """Raised when the session does not have permission to take some action""" faultCode = 1004 -## BEGIN kojikamid dup +# BEGIN kojikamid dup # class BuildError(GenericError): """Raised when a build fails""" faultCode = 1005 -## END kojikamid dup +# END kojikamid dup # class AuthLockError(AuthError): """Raised when a lock prevents authentication""" @@ -403,7 +403,7 @@ class MultiCallInProgress(object): pass -#A function to get create an exception from a fault +# A function to get create an exception from a fault def convertFault(fault): """Convert a fault to the corresponding Exception type, if possible""" code = getattr(fault, 'faultCode', None) @@ -415,7 +415,7 @@ def convertFault(fault): ret = v(fault.faultString) ret.fromFault = True return ret - #otherwise... + # otherwise... return fault def listFaults(): @@ -440,7 +440,7 @@ def listFaults(): ret.sort(key=lambda x: x['faultCode']) return ret -#functions for encoding/decoding optional arguments +# functions for encoding/decoding optional arguments def encode_args(*args, **opts): """The function encodes optional arguments as regular arguments. @@ -481,10 +481,10 @@ def decode_int(n): """If n is not an integer, attempt to convert it""" if isinstance(n, six.integer_types): return n - #else + # else return int(n) -#commonly used functions +# commonly used functions def safe_xmlrpc_loads(s): """Load xmlrpc data from a string, but catch faults""" @@ -493,7 +493,7 @@ def safe_xmlrpc_loads(s): except Fault as f: return f -## BEGIN kojikamid dup +# BEGIN kojikamid dup # def ensuredir(directory): @@ -528,7 +528,7 @@ def ensuredir(directory): raise return directory -## END kojikamid dup +# END kojikamid dup # def daemonize(): """Detach and run in background""" @@ -537,12 +537,12 @@ def daemonize(): os._exit(0) os.setsid() signal.signal(signal.SIGHUP, signal.SIG_IGN) - #fork again + # fork again pid = os.fork() if pid: os._exit(0) os.chdir("/") - #redirect stdin/stdout/sterr + # redirect stdin/stdout/sterr fd0 = os.open('/dev/null', os.O_RDONLY) fd1 = os.open('/dev/null', os.O_RDWR) fd2 = os.open('/dev/null', os.O_RDWR) @@ -597,7 +597,7 @@ def rpm_hdr_size(f, ofs=None): il = multibyte(data[0:4]) dl = multibyte(data[4:8]) - #this is what the section data says the size should be + # this is what the section data says the size should be hdrsize = 8 + 16 * il + dl # hdrsize rounded up to nearest 8 bytes @@ -624,7 +624,7 @@ class RawHeader(object): self._index() def version(self): - #fourth byte is the version + # fourth byte is the version return _ord(self.header[3]) def _index(self): @@ -635,7 +635,7 @@ class RawHeader(object): il = multibyte(data[:4]) dl = multibyte(data[4:8]) - #read the index (starts at offset 16) + # read the index (starts at offset 16) index = {} for i in range(il): entry = [] @@ -643,30 +643,31 @@ class RawHeader(object): ofs = 16 + i*16 + j*4 data = [_ord(x) for x in self.header[ofs:ofs+4]] entry.append(multibyte(data)) - #print("Tag: %d, Type: %d, Offset: %x, Count: %d" % tuple(entry)) + + # print("Tag: %d, Type: %d, Offset: %x, Count: %d" % tuple(entry)) index[entry[0]] = entry self.datalen = dl self.index = index def dump(self): print("HEADER DUMP:") - #calculate start of store + # calculate start of store il = len(self.index) store = 16 + il * 16 - #print("start is: %d" % start) - #print("index length: %d" % il) + # print("start is: %d" % start) + # print("index length: %d" % il) print("Store at offset %d (%0x)" % (store, store)) - #sort entries by offset, dtype - #also rearrange: tag, dtype, offset, count -> offset, dtype, tag, count + # sort entries by offset, dtype + # also rearrange: tag, dtype, offset, count -> offset, dtype, tag, count order = sorted([(x[2], x[1], x[0], x[3]) for x in six.itervalues(self.index)]) next = store - #map some rpmtag codes + # map some rpmtag codes tags = {} for name, code in six.iteritems(rpm.__dict__): if name.startswith('RPMTAG_') and isinstance(code, int): tags[code] = name[7:].lower() for entry in order: - #tag, dtype, offset, count = entry + # tag, dtype, offset, count = entry offset, dtype, tag, count = entry pos = store + offset if next is not None: @@ -679,17 +680,17 @@ class RawHeader(object): print("Tag: %d [%s], Type: %d, Offset: %x, Count: %d" \ % (tag, tags.get(tag, '?'), dtype, offset, count)) if dtype == 0: - #null + # null print("[NULL entry]") next = pos elif dtype == 1: - #char + # char for i in range(count): print("Char: %r" % self.header[pos]) pos += 1 next = pos elif dtype >= 2 and dtype <= 5: - #integer + # integer n = 1 << (dtype - 2) for i in range(count): data = [_ord(x) for x in self.header[pos:pos+n]] @@ -738,7 +739,7 @@ class RawHeader(object): return self._getitem(dtype, offset, count) def _getitem(self, dtype, offset, count): - #calculate start of store + # calculate start of store il = len(self.index) store = 16 + il * 16 pos = store + offset @@ -752,10 +753,10 @@ class RawHeader(object): end = self.header.find('\0', pos) return self.header[pos:end] elif dtype == 7: - #raw data + # raw data return self.header[pos:pos+count] else: - #XXX - not all valid data types are handled + # XXX - not all valid data types are handled raise GenericError("Unable to read header data type: %x" % dtype) def get(self, key, default=None): @@ -1108,7 +1109,7 @@ def is_debuginfo(name): def canonArch(arch): """Given an arch, return the "canonical" arch""" - #XXX - this could stand to be smarter, and we should probably + # XXX - this could stand to be smarter, and we should probably # have some other related arch-mangling functions. if fnmatch(arch, 'i?86') or arch == 'athlon': return 'i386' @@ -1295,12 +1296,12 @@ BuildArch: noarch #package requirements """] - #add a requires entry for all the packages in buildgroup, and in - #groups required by buildgroup + # add a requires entry for all the packages in buildgroup, and in + # groups required by buildgroup need = [buildgroup] seen_grp = {} seen_pkg = {} - #index groups + # index groups groups = dict([(g['name'], g) for g in grplist]) for group_name in need: if group_name in seen_grp: @@ -1375,7 +1376,7 @@ def generate_comps(groups, expand_groups=False): """ %s """ % boolean_text(True)) - #print grouplist, if any + # print grouplist, if any if g['grouplist'] and not expand_groups: data.append( """ @@ -1383,7 +1384,7 @@ def generate_comps(groups, expand_groups=False): grouplist = list(g['grouplist']) grouplist.sort(key=lambda x: x['name']) for x in grouplist: - #['req_id','type','is_metapkg','name'] + # ['req_id','type','is_metapkg','name'] name = x['name'] thetype = x['type'] tag = "groupreq" @@ -1401,9 +1402,9 @@ def generate_comps(groups, expand_groups=False): """ """) - #print packagelist, if any + # print packagelist, if any def package_entry(pkg): - #p['package_id','type','basearchonly','requires','name'] + # p['package_id','type','basearchonly','requires','name'] name = pkg['package'] opts = 'type="%s"' % pkg['type'] if pkg['basearchonly']: @@ -1424,7 +1425,7 @@ def generate_comps(groups, expand_groups=False): """ % package_entry(p)) # also include expanded list, if needed if expand_groups and g['grouplist']: - #add a requires entry for all packages in groups required by buildgroup + # add a requires entry for all packages in groups required by buildgroup need = [req['name'] for req in g['grouplist']] seen_grp = {g['name'] : 1} seen_pkg = {} @@ -1484,12 +1485,12 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) raise GenericError("please provide a repo and tag") topurls = opts.get('topurls') if not topurls: - #cli command still passes plain topurl + # cli command still passes plain topurl topurl = opts.get('topurl') if topurl: topurls = [topurl] if topurls: - #XXX - PathInfo isn't quite right for this, but it will do for now + # XXX - PathInfo isn't quite right for this, but it will do for now pathinfos = [PathInfo(topdir=_u) for _u in topurls] urls = ["%s/%s" % (_p.repo(repoid, tag_name), arch) for _p in pathinfos] else: @@ -1539,7 +1540,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) if mavenrc: files['etc/mavenrc'] = mavenrc - #generate yum.conf + # generate yum.conf yc_parts = ["[main]\n"] # HTTP proxy for yum if opts.get('yum_proxy'): @@ -1780,7 +1781,7 @@ def read_config(profile_name, user_config=None): result = config_defaults.copy() - #note: later config files override earlier ones + # note: later config files override earlier ones # /etc/koji.conf.d configs = ['/etc/koji.conf.d'] @@ -1807,9 +1808,9 @@ def read_config(profile_name, user_config=None): got_conf = True result['profile'] = profile_name for name, value in config.items(profile_name): - #note the config_defaults dictionary also serves to indicate which - #options *can* be set via the config file. Such options should - #not have a default value set in the option parser. + # note the config_defaults dictionary also serves to indicate which + # options *can* be set via the config file. Such options should + # not have a default value set in the option parser. if name in result: if name in ('anon_retry', 'offline_retry', 'use_fast_upload', 'krb_rdns', 'debug', @@ -1984,7 +1985,7 @@ class PathInfo(object): def volumedir(self, volume): if volume == 'DEFAULT' or volume is None: return self.topdir - #else + # else return self.topdir + ("/vol/%s" % volume) def build(self, build): @@ -2141,7 +2142,7 @@ def is_cert_error(e): 'certificate expired' in ssl_reason): return True - #otherwise + # otherwise return False @@ -2553,7 +2554,7 @@ class ClientSession(object): handler, headers, request = self._prepCall('logout', ()) self._sendCall(handler, headers, request) except AuthExpired: - #this can happen when an exclusive session is forced + # this can happen when an exclusive session is forced pass self.setSession(None) @@ -2578,10 +2579,10 @@ class ClientSession(object): return self.setSession(None) - #we've had some trouble with this method causing strange problems - #(like infinite recursion). Possibly triggered by initialization failure, - #and possibly due to some interaction with __getattr__. - #Re-enabling with a small improvement + # we've had some trouble with this method causing strange problems + # (like infinite recursion). Possibly triggered by initialization failure, + # and possibly due to some interaction with __getattr__. + # Re-enabling with a small improvement def __del__(self): if self.__dict__: try: @@ -2594,7 +2595,7 @@ class ClientSession(object): return self._callMethod(name, args, opts) def _prepCall(self, name, args, kwargs=None): - #pass named opts in a way the server can understand + # pass named opts in a way the server can understand if kwargs is None: kwargs = {} if name == 'rawUpload': @@ -2713,27 +2714,27 @@ class ClientSession(object): self.retries += 1 try: return self._sendCall(handler, headers, request) - #basically, we want to retry on most errors, with a few exceptions + # basically, we want to retry on most errors, with a few exceptions # - faults (this means the call completed and failed) # - SystemExit, KeyboardInterrupt # note that, for logged-in sessions the server should tell us (via a RetryError fault) # if the call cannot be retried. For non-logged-in sessions, all calls should be read-only # and hence retryable. except Fault as fault: - #try to convert the fault to a known exception + # try to convert the fault to a known exception err = convertFault(fault) if isinstance(err, ServerOffline): if self.opts.get('offline_retry', False): secs = self.opts.get('offline_retry_interval', interval) self.logger.debug("Server offline. Retrying in %i seconds", secs) time.sleep(secs) - #reset try count - this isn't a typical error, this is a running server - #correctly reporting an outage + # reset try count - this isn't a typical error, this is a running server + # correctly reporting an outage tries = 0 continue raise err except (SystemExit, KeyboardInterrupt): - #(depending on the python version, these may or may not be subclasses of Exception) + # (depending on the python version, these may or may not be subclasses of Exception) raise except Exception as e: tb_str = ''.join(traceback.format_exception(*sys.exc_info())) @@ -2744,8 +2745,8 @@ class ClientSession(object): raise if not self.logged_in: - #in the past, non-logged-in sessions did not retry. For compatibility purposes - #this behavior is governed by the anon_retry opt. + # in the past, non-logged-in sessions did not retry. For compatibility purposes + # this behavior is governed by the anon_retry opt. if not self.opts.get('anon_retry', False): raise @@ -2754,14 +2755,14 @@ class ClientSession(object): if tries > max_retries: raise - #otherwise keep retrying + # otherwise keep retrying if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug(tb_str) self.logger.info("Try #%s for call %s (%s) failed: %s", tries, self.callnum, name, e) if tries > 1: # first retry is immediate, after that we honor retry_interval time.sleep(interval) - #not reached + # not reached def multiCall(self, strict=False, batch=None): """Execute a prepared multicall @@ -2816,7 +2817,7 @@ class ClientSession(object): else: ret = self._callMethod('multiCall', (calls,), {}) if strict: - #check for faults and raise first one + # check for faults and raise first one for entry in ret: if isinstance(entry, dict): fault = Fault(entry['faultCode'], entry['faultString']) @@ -2825,7 +2826,7 @@ class ClientSession(object): return ret def __getattr__(self, name): - #if name[:1] == '_': + # if name[:1] == '_': # raise AttributeError("no attribute %r" % name) if name == '_apidoc': return self.__dict__['_apidoc'] @@ -2953,7 +2954,7 @@ class ClientSession(object): start = time.time() # XXX - stick in a config or something retries = 3 - fo = open(localfile, "rb") #specify bufsize? + fo = open(localfile, "rb") # specify bufsize? totalsize = os.path.getsize(localfile) ofs = 0 md5sum = hashlib.md5() @@ -3207,15 +3208,15 @@ class DBHandler(logging.Handler): columns.append(key) values.append("%%(%s)s" % key) data[key] = value % record.__dict__ - #values.append(_quote(value % record.__dict__)) + # values.append(_quote(value % record.__dict__)) columns = ",".join(columns) values = ",".join(values) command = "INSERT INTO %s (%s) VALUES (%s)" % (self.table, columns, values) - #note we're letting cursor.execute do the escaping + # note we're letting cursor.execute do the escaping cursor.execute(command, data) cursor.close() - #self.cnx.commit() - #XXX - committing here is most likely wrong, but we need to set commit_pending or something + # self.cnx.commit() + # XXX - committing here is most likely wrong, but we need to set commit_pending or something # ...and this is really the wrong place for that except: self.handleError(record) @@ -3328,7 +3329,7 @@ def _taskLabel(taskInfo): extra = build_target['name'] elif method == 'winbuild': if 'request' in taskInfo: - #vm = taskInfo['request'][0] + # vm = taskInfo['request'][0] url = taskInfo['request'][1] target = taskInfo['request'][2] module_info = _module_info(url) diff --git a/koji/arch.py b/koji/arch.py index 431c9bc..a36046f 100644 --- a/koji/arch.py +++ b/koji/arch.py @@ -33,7 +33,7 @@ arches = { "amd64": "x86_64", "ia32e": "x86_64", - #ppc64le + # ppc64le "ppc64le": "noarch", # ppc @@ -73,7 +73,7 @@ arches = { "armv5tejl": "armv5tel", "armv5tel": "noarch", - #arm hardware floating point + # arm hardware floating point "armv7hnl": "armv7hl", "armv7hl": "armv6hl", "armv6hl": "noarch", @@ -86,7 +86,7 @@ arches = { "sh4": "noarch", "sh3": "noarch", - #itanium + # itanium "ia64": "noarch", } diff --git a/koji/auth.py b/koji/auth.py index 8520069..c2a0bdd 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -79,7 +79,7 @@ class Session(object): self._perms = None self._groups = None self._host_id = '' - #get session data from request + # get session data from request if args is None: environ = getattr(context, 'environ', {}) args = environ.get('QUERY_STRING', '') @@ -97,7 +97,7 @@ class Session(object): callnum = args['callnum'][0] except: callnum = None - #lookup the session + # lookup the session c = context.cnx.cursor() fields = { 'authtype': 'authtype', @@ -125,10 +125,10 @@ class Session(object): if not row: raise koji.AuthError('Invalid session or bad credentials') session_data = dict(zip(aliases, row)) - #check for expiration + # check for expiration if session_data['expired']: raise koji.AuthExpired('session "%i" has expired' % id) - #check for callnum sanity + # check for callnum sanity if callnum is not None: try: callnum = int(callnum) @@ -140,14 +140,14 @@ class Session(object): raise koji.SequenceError("%d > %d (session %d)" \ % (lastcall, callnum, id)) elif lastcall == callnum: - #Some explanation: - #This function is one of the few that performs its own commit. - #However, our storage of the current callnum is /after/ that - #commit. This means the the current callnum only gets committed if - #a commit happens afterward. - #We only schedule a commit for dml operations, so if we find the - #callnum in the db then a previous attempt succeeded but failed to - #return. Data was changed, so we cannot simply try the call again. + # Some explanation: + # This function is one of the few that performs its own commit. + # However, our storage of the current callnum is /after/ that + # commit. This means the the current callnum only gets committed if + # a commit happens afterward. + # We only schedule a commit for dml operations, so if we find the + # callnum in the db then a previous attempt succeeded but failed to + # return. Data was changed, so we cannot simply try the call again. method = getattr(context, 'method', 'UNKNOWN') if method not in RetryWhitelist: raise koji.RetryError( @@ -155,7 +155,7 @@ class Session(object): % (callnum, method, id)) # read user data - #historical note: + # historical note: # we used to get a row lock here as an attempt to maintain sanity of exclusive # sessions, but it was an imperfect approach and the lock could cause some # performance issues. @@ -166,25 +166,25 @@ class Session(object): if user_data['status'] != koji.USER_STATUS['NORMAL']: raise koji.AuthError('logins by %s are not allowed' % user_data['name']) - #check for exclusive sessions + # check for exclusive sessions if session_data['exclusive']: - #we are the exclusive session for this user + # we are the exclusive session for this user self.exclusive = True else: - #see if an exclusive session exists + # see if an exclusive session exists q = """SELECT id FROM sessions WHERE user_id=%(user_id)s AND "exclusive" = TRUE AND expired = FALSE""" - #should not return multiple rows (unique constraint) + # should not return multiple rows (unique constraint) c.execute(q, session_data) row = c.fetchone() if row: (excl_id,) = row if excl_id == session_data['master']: - #(note excl_id cannot be None) - #our master session has the lock + # (note excl_id cannot be None) + # our master session has the lock self.exclusive = True else: - #a session unrelated to us has the lock + # a session unrelated to us has the lock self.lockerror = "User locked by another session" # we don't enforce here, but rely on the dispatcher to enforce # if appropriate (otherwise it would be impossible to steal @@ -193,11 +193,11 @@ class Session(object): # update timestamp q = """UPDATE sessions SET update_time=NOW() WHERE id = %(id)i""" c.execute(q, locals()) - #save update time + # save update time context.cnx.commit() - #update callnum (this is deliberately after the commit) - #see earlier note near RetryError + # update callnum (this is deliberately after the commit) + # see earlier note near RetryError if callnum is not None: q = """UPDATE sessions SET callnum=%(callnum)i WHERE id = %(id)i""" c.execute(q, locals()) @@ -218,7 +218,7 @@ class Session(object): # grab perm and groups data on the fly if name == 'perms': if self._perms is None: - #in a dict for quicker lookup + # in a dict for quicker lookup self._perms = dict([[name, 1] for name in get_user_perms(self.user_id)]) return self._perms elif name == 'groups': @@ -254,7 +254,7 @@ class Session(object): return override else: hostip = context.environ['REMOTE_ADDR'] - #XXX - REMOTE_ADDR not promised by wsgi spec + # XXX - REMOTE_ADDR not promised by wsgi spec if hostip == '127.0.0.1': hostip = socket.gethostbyname(socket.gethostname()) return hostip @@ -294,7 +294,7 @@ class Session(object): self.checkLoginAllowed(user_id) - #create session and return + # create session and return sinfo = self.createSession(user_id, hostip, koji.AUTHTYPE_NORMAL) session_id = sinfo['session-id'] context.cnx.commit() @@ -386,7 +386,7 @@ class Session(object): # so get the local ip via a different method local_ip = socket.gethostbyname(context.environ['SERVER_NAME']) remote_ip = context.environ['REMOTE_ADDR'] - #XXX - REMOTE_ADDR not promised by wsgi spec + # XXX - REMOTE_ADDR not promised by wsgi spec # it appears that calling setports() with *any* value results in authentication # failing with "Incorrect net address", so return 0 (which prevents @@ -466,11 +466,11 @@ class Session(object): if self.master is not None: raise koji.GenericError("subsessions cannot become exclusive") if self.exclusive: - #shouldn't happen + # shouldn't happen raise koji.GenericError("session is already exclusive") user_id = self.user_id session_id = self.id - #acquire a row lock on the user entry + # acquire a row lock on the user entry q = """SELECT id FROM users WHERE id=%(user_id)s FOR UPDATE""" c.execute(q, locals()) # check that no other sessions for this user are exclusive @@ -481,13 +481,13 @@ class Session(object): row = c.fetchone() if row: if force: - #expire the previous exclusive session and try again + # expire the previous exclusive session and try again (excl_id,) = row q = """UPDATE sessions SET expired=TRUE,"exclusive"=NULL WHERE id=%(excl_id)s""" c.execute(q, locals()) else: raise koji.AuthLockError("Cannot get exclusive session") - #mark this session exclusive + # mark this session exclusive q = """UPDATE sessions SET "exclusive"=TRUE WHERE id=%(session_id)s""" c.execute(q, locals()) context.cnx.commit() @@ -503,12 +503,12 @@ class Session(object): def logout(self): """expire a login session""" if not self.logged_in: - #XXX raise an error? + # XXX raise an error? raise koji.AuthError("Not logged in") update = """UPDATE sessions SET expired=TRUE,exclusive=NULL WHERE id = %(id)i OR master = %(id)i""" - #note we expire subsessions as well + # note we expire subsessions as well c = context.cnx.cursor() c.execute(update, {'id': self.id}) context.cnx.commit() @@ -517,7 +517,7 @@ class Session(object): def logoutChild(self, session_id): """expire a subsession""" if not self.logged_in: - #XXX raise an error? + # XXX raise an error? raise koji.AuthError("Not logged in") update = """UPDATE sessions SET expired=TRUE,exclusive=NULL @@ -547,7 +547,7 @@ class Session(object): (session_id,) = c.fetchone() - #add session id to database + # add session id to database q = """ INSERT INTO sessions (id, user_id, key, hostip, authtype, master) VALUES (%(session_id)i, %(user_id)i, %(key)s, %(hostip)s, %(authtype)i, %(master)s) @@ -555,7 +555,7 @@ class Session(object): c.execute(q, locals()) context.cnx.commit() - #return session info + # return session info return {'session-id' : session_id, 'session-key' : key} def subsession(self): @@ -589,7 +589,7 @@ class Session(object): def hasGroup(self, group_id): if not self.logged_in: return False - #groups indexed by id + # groups indexed by id return group_id in self.groups def isUser(self, user_id): @@ -616,7 +616,7 @@ class Session(object): return None def getHostId(self): - #for compatibility + # for compatibility return self.host_id def getUserId(self, username): @@ -805,7 +805,7 @@ def get_user_perms(user_id): FROM user_perms JOIN permissions ON perm_id = permissions.id WHERE active = TRUE AND user_id=%(user_id)s""" c.execute(q, locals()) - #return a list of permissions by name + # return a list of permissions by name return [row[0] for row in c.fetchall()] def get_user_data(user_id): diff --git a/koji/daemon.py b/koji/daemon.py index 8bb3a2e..9f5fb56 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -171,7 +171,7 @@ def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, a return status[1] -## BEGIN kojikamid dup +# BEGIN kojikamid dup # class SCM(object): "SCM abstraction class" @@ -397,7 +397,7 @@ class SCM(object): env = None def _run(cmd, chdir=None, fatal=False, log=True, _count=[0]): if globals().get('KOJIKAMID'): - #we've been inserted into kojikamid, use its run() + # we've been inserted into kojikamid, use its run() return run(cmd, chdir=chdir, fatal=fatal, log=log) # noqa: F821 else: append = (_count[0] > 0) @@ -546,7 +546,7 @@ class SCM(object): # just use the same url r['source'] = self.url return r -## END kojikamid dup +# END kojikamid dup # class TaskManager(object): @@ -613,7 +613,7 @@ class TaskManager(object): If nolocal is True, do not try to scan local buildroots. """ - #query buildroots in db that are not expired + # query buildroots in db that are not expired states = [koji.BR_STATES[x] for x in ('INIT', 'WAITING', 'BUILDING')] db_br = self.session.listBuildroots(hostID=self.host_id, state=tuple(states)) # index by id @@ -627,8 +627,8 @@ class TaskManager(object): self.logger.warn("Expiring taskless buildroot: %(id)i/%(tag_name)s/%(arch)s" % br) self.session.host.setBuildRootState(id, st_expired) elif task_id not in self.tasks: - #task not running - expire the buildroot - #TODO - consider recycling hooks here (with strong sanity checks) + # task not running - expire the buildroot + # TODO - consider recycling hooks here (with strong sanity checks) self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br) self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, to_list(self.tasks.keys()))) self.session.host.setBuildRootState(id, st_expired) @@ -640,13 +640,13 @@ class TaskManager(object): local_only = [id for id in local_br if id not in db_br] if local_only: missed_br = self.session.listBuildroots(buildrootID=tuple(local_only)) - #get all the task info in one call + # get all the task info in one call tasks = [] for br in missed_br: task_id = br['task_id'] if task_id: tasks.append(task_id) - #index + # index missed_br = dict([(row['id'], row) for row in missed_br]) tasks = dict([(row['id'], row) for row in self.session.getTaskInfo(tasks)]) for id in local_only: @@ -671,7 +671,7 @@ class TaskManager(object): self.logger.warn("%s: invalid task %s" % (desc, br['task_id'])) continue if (task['state'] == koji.TASK_STATES['FAILED'] and age < self.options.failed_buildroot_lifetime): - #XXX - this could be smarter + # XXX - this could be smarter # keep buildroots for failed tasks around for a little while self.logger.debug("Keeping failed buildroot: %s" % desc) continue @@ -689,17 +689,17 @@ class TaskManager(object): continue else: age = min(age, time.time() - st.st_mtime) - #note: https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=192153) - #If rpmlib is installing in this chroot, removing it entirely - #can lead to a world of hurt. - #We remove the rootdir contents but leave the rootdir unless it - #is really old + # note: https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=192153) + # If rpmlib is installing in this chroot, removing it entirely + # can lead to a world of hurt. + # We remove the rootdir contents but leave the rootdir unless it + # is really old if age > 3600*24: - #dir untouched for a day + # dir untouched for a day self.logger.info("Removing buildroot: %s" % desc) if topdir and safe_rmtree(topdir, unmount=True, strict=False) != 0: continue - #also remove the config + # also remove the config try: os.unlink(data['cfg']) except OSError as e: @@ -726,7 +726,7 @@ class TaskManager(object): self.logger.debug("Expired/stray buildroots: %d" % len(local_only)) def _scanLocalBuildroots(self): - #XXX + # XXX configdir = '/etc/mock/koji' buildroots = {} for f in os.listdir(configdir): @@ -785,13 +785,13 @@ class TaskManager(object): # by this host. id = task['id'] if id not in self.pids: - #We don't have a process for this - #Expected to happen after a restart, otherwise this is an error + # We don't have a process for this + # Expected to happen after a restart, otherwise this is an error stale.append(id) continue tasks[id] = task if task.get('alert', False): - #wake up the process + # wake up the process self.logger.info("Waking up task: %r" % task) os.kill(self.pids[id], signal.SIGUSR2) if not task['waiting']: @@ -801,8 +801,8 @@ class TaskManager(object): self.tasks = tasks self.logger.debug("Current tasks: %r" % self.tasks) if len(stale) > 0: - #A stale task is one which is opened to us, but we know nothing - #about). This will happen after a daemon restart, for example. + # A stale task is one which is opened to us, but we know nothing + # about). This will happen after a daemon restart, for example. self.logger.info("freeing stale tasks: %r" % stale) self.session.host.freeTasks(stale) for id, pid in list(self.pids.items()): @@ -844,15 +844,15 @@ class TaskManager(object): self.logger.debug("Load Data:") self.logger.debug(" hosts: %r" % hosts) self.logger.debug(" tasks: %r" % tasks) - #now we organize this data into channel-arch bins + # now we organize this data into channel-arch bins bin_hosts = {} #hosts indexed by bin bins = {} #bins for this host our_avail = None for host in hosts: host['bins'] = [] if host['id'] == self.host_id: - #note: task_load reported by server might differ from what we - #sent due to precision variation + # note: task_load reported by server might differ from what we + # sent due to precision variation our_avail = host['capacity'] - host['task_load'] for chan in host['channels']: for arch in host['arches'].split() + ['noarch']: @@ -867,7 +867,7 @@ class TaskManager(object): elif not bins: self.logger.info("No bins for this host. Missing channel/arch config?") # Note: we may still take an assigned task below - #sort available capacities for each of our bins + # sort available capacities for each of our bins avail = {} for bin in bins: avail[bin] = [host['capacity'] - host['task_load'] for host in bin_hosts[bin]] @@ -889,7 +889,7 @@ class TaskManager(object): if task['state'] == koji.TASK_STATES['ASSIGNED']: self.logger.debug("task is assigned") if self.host_id == task['host_id']: - #assigned to us, we can take it regardless + # assigned to us, we can take it regardless if self.takeTask(task): return True elif task['state'] == koji.TASK_STATES['FREE']: @@ -897,18 +897,18 @@ class TaskManager(object): self.logger.debug("task is free, bin=%r" % bin) if bin not in bins: continue - #see where our available capacity is compared to other hosts for this bin - #(note: the hosts in this bin are exactly those that could - #accept this task) + # see where our available capacity is compared to other hosts for this bin + # (note: the hosts in this bin are exactly those that could + # accept this task) bin_avail = avail.get(bin, [0]) if self.checkAvailDelay(task, bin_avail, our_avail): # decline for now and give the upper half a chance continue - #otherwise, we attempt to open the task + # otherwise, we attempt to open the task if self.takeTask(task): return True else: - #should not happen + # should not happen raise Exception("Invalid task state reported by server") return False @@ -968,11 +968,11 @@ class TaskManager(object): try: (childpid, status) = os.waitpid(pid, os.WNOHANG) except OSError as e: - #check errno + # check errno if e.errno != errno.ECHILD: - #should not happen + # should not happen raise - #otherwise assume the process is gone + # otherwise assume the process is gone self.logger.info("%s: %s" % (prefix, e)) return True if childpid != 0: @@ -1118,7 +1118,7 @@ class TaskManager(object): if children: self._killChildren(task_id, children, sig=signal.SIGKILL, timeout=3.0) - #expire the task's subsession + # expire the task's subsession session_id = self.subsessions.get(task_id) if session_id: self.logger.info("Expiring subsession %i (task %i)" % (session_id, task_id)) @@ -1126,7 +1126,7 @@ class TaskManager(object): self.session.logoutChild(session_id) del self.subsessions[task_id] except: - #not much we can do about it + # not much we can do about it pass if wait: return self._waitTask(task_id, pid) @@ -1200,7 +1200,7 @@ class TaskManager(object): self.status = "Load average %.2f > %.2f" % (loadavgs[0], maxload) self.logger.info(self.status) return False - #XXX - add more checks + # XXX - add more checks return True def takeTask(self, task): @@ -1250,7 +1250,7 @@ class TaskManager(object): if state != 'OPEN': self.logger.warn("Task %i changed is %s", task_id, state) return False - #otherwise... + # otherwise... raise if handler.Foreground: self.logger.info("running task in foreground") @@ -1263,27 +1263,27 @@ class TaskManager(object): return True def forkTask(self, handler): - #get the subsession before we fork + # get the subsession before we fork newhub = self.session.subsession() session_id = newhub.sinfo['session-id'] pid = os.fork() if pid: newhub._forget() return pid, session_id - #in no circumstance should we return after the fork - #nor should any exceptions propagate past here + # in no circumstance should we return after the fork + # nor should any exceptions propagate past here try: self.session._forget() - #set process group + # set process group os.setpgrp() - #use the subsession + # use the subsession self.session = newhub handler.session = self.session - #set a do-nothing handler for sigusr2 + # set a do-nothing handler for sigusr2 signal.signal(signal.SIGUSR2, lambda *args: None) self.runTask(handler) finally: - #diediedie + # diediedie try: self.session.logout() finally: @@ -1302,10 +1302,10 @@ class TaskManager(object): tb = ''.join(traceback.format_exception(*sys.exc_info())).replace(r"\n", "\n") self.logger.warn("FAULT:\n%s" % tb) except (SystemExit, koji.tasks.ServerExit, KeyboardInterrupt): - #we do not trap these + # we do not trap these raise except koji.tasks.ServerRestart: - #freeing this task will allow the pending restart to take effect + # freeing this task will allow the pending restart to take effect self.session.host.freeTasks([handler.id]) return except: @@ -1315,7 +1315,7 @@ class TaskManager(object): e_class, e = sys.exc_info()[:2] faultCode = getattr(e_class, 'faultCode', 1) if issubclass(e_class, koji.GenericError): - #just pass it through + # just pass it through tb = str(e) response = koji.xmlrpcplus.dumps(koji.xmlrpcplus.Fault(faultCode, tb)) diff --git a/koji/db.py b/koji/db.py index 8a085bf..3d36b87 100644 --- a/koji/db.py +++ b/koji/db.py @@ -75,8 +75,8 @@ class DBWrapper: if not self.cnx: raise Exception('connection is closed') self.cnx.cursor().execute('ROLLBACK') - #We do this rather than cnx.rollback to avoid opening a new transaction - #If our connection gets recycled cnx.rollback will be called then. + # We do this rather than cnx.rollback to avoid opening a new transaction + # If our connection gets recycled cnx.rollback will be called then. self.cnx = None @@ -177,7 +177,7 @@ def connect(): return DBWrapper(conn) except psycopg2.Error: del _DBconn.conn - #create a fresh connection + # create a fresh connection opts = _DBopts if opts is None: opts = {} diff --git a/koji/plugin.py b/koji/plugin.py index d6e8555..e8183de 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -62,7 +62,7 @@ class PluginTracker(object): def __init__(self, path=None, prefix='_koji_plugin__'): self.searchpath = path - #prefix should not have a '.' in it, this can cause problems. + # prefix should not have a '.' in it, this can cause problems. self.prefix = prefix self.plugins = {} @@ -71,9 +71,9 @@ class PluginTracker(object): return self.plugins[name] mod_name = name if self.prefix: - #mod_name determines how the module is named in sys.modules - #Using a prefix helps prevent overlap with other modules - #(no '.' -- it causes problems) + # mod_name determines how the module is named in sys.modules + # Using a prefix helps prevent overlap with other modules + # (no '.' -- it causes problems) mod_name = self.prefix + name if mod_name in sys.modules and not reload: raise koji.PluginError('module name conflict: %s' % mod_name) diff --git a/koji/policy.py b/koji/policy.py index fa1f988..27dbdf6 100644 --- a/koji/policy.py +++ b/koji/policy.py @@ -31,7 +31,7 @@ from koji.util import to_list class BaseSimpleTest(object): """Abstract base class for simple tests""" - #Provide the name of the test + # Provide the name of the test name = None def __init__(self, str): @@ -62,12 +62,12 @@ class FalseTest(BaseSimpleTest): class AllTest(TrueTest): name = 'all' - #alias for true + # alias for true class NoneTest(FalseTest): name = 'none' - #alias for false + # alias for false class HasTest(BaseSimpleTest): @@ -233,11 +233,11 @@ class SimpleRuleSet(object): for line in lines: rule = self.parse_line(line) if rule is None: - #blank/etc + # blank/etc continue tests, negate, action = rule if action == '{': - #nested rules + # nested rules child = [] cursor.append([tests, negate, child]) stack.append(cursor) @@ -275,11 +275,11 @@ class SimpleRuleSet(object): """ line = line.split('#', 1)[0].strip() if not line: - #blank or all comment + # blank or all comment return None if line == '}': return None, False, '}' - #?? allow }} ?? + # ?? allow }} ?? negate = False pos = line.rfind('::') if pos == -1: @@ -328,7 +328,7 @@ class SimpleRuleSet(object): if not check: break else: - #all tests in current rule passed + # all tests in current rule passed value = True if negate: value = not value @@ -393,11 +393,11 @@ def findSimpleTests(namespace): if isinstance(value, type(BaseSimpleTest)) and issubclass(value, BaseSimpleTest): name = getattr(value, 'name', None) if not name: - #use the class name + # use the class name name = key - #but trim 'Test' from the end + # but trim 'Test' from the end if name.endswith('Test') and len(name) > 4: name = name[:-4] ret.setdefault(name, value) - #...so first test wins in case of name overlap + # ...so first test wins in case of name overlap return ret diff --git a/koji/rpmdiff.py b/koji/rpmdiff.py index 12f72af..efa2dfc 100644 --- a/koji/rpmdiff.py +++ b/koji/rpmdiff.py @@ -48,7 +48,7 @@ class Rpmdiff: PRCO = ( 'REQUIRES', 'PROVIDES', 'CONFLICTS', 'OBSOLETES') - #{fname : (size, mode, mtime, flags, dev, inode, + # {fname : (size, mode, mtime, flags, dev, inode, # nlink, state, vflags, user, group, digest)} __FILEIDX = [ ['S', 0], ['M', 1], @@ -71,7 +71,7 @@ class Rpmdiff: try: PREREQ_FLAG=rpm.RPMSENSE_PREREQ except: - #(proyvind): This seems ugly, but then again so does + # (proyvind): This seems ugly, but then again so does # this whole check as well. PREREQ_FLAG=False diff --git a/koji/tasks.py b/koji/tasks.py index af603a0..a86d0a6 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -51,7 +51,7 @@ def scan_mounts(topdir): logger.warning('Found deleted mountpoint: %s' % path) mplist.append(path) fo.close() - #reverse sort so deeper dirs come first + # reverse sort so deeper dirs come first mplist.sort(reverse=True) return mplist @@ -64,7 +64,7 @@ def umount_all(topdir): rv = os.spawnvp(os.P_WAIT, cmd[0], cmd) if rv != 0: raise koji.GenericError('umount failed (exit code %r) for %s' % (rv, path)) - #check mounts again + # check mounts again remain = scan_mounts(topdir) if remain: raise koji.GenericError("Unmounting incomplete: %r" % remain) @@ -340,7 +340,7 @@ class BaseTaskHandler(object): if self.workdir is None: return safe_rmtree(self.workdir, unmount=False, strict=True) - #os.spawnvp(os.P_WAIT, 'rm', ['rm', '-rf', self.workdir]) + # os.spawnvp(os.P_WAIT, 'rm', ['rm', '-rf', self.workdir]) def wait(self, subtasks=None, all=False, failany=False, canfail=None, timeout=None): @@ -385,7 +385,7 @@ class BaseTaskHandler(object): while True: finished, unfinished = self.session.host.taskWait(self.id) if len(unfinished) == 0: - #all done + # all done break elif len(finished) > 0: if all: @@ -561,7 +561,7 @@ class BaseTaskHandler(object): repo_info = self.session.getRepo(tag) taginfo = self.session.getTag(tag, strict=True) if not repo_info: - #make sure there is a target + # make sure there is a target targets = self.session.getBuildTargets(buildTagID=taginfo['id']) if not targets: raise koji.BuildError('no repo (and no target) for tag %s' % taginfo['name']) @@ -666,7 +666,7 @@ class ShutdownTask(BaseTaskHandler): _taskWeight = 0.0 Foreground = True def handler(self): - #note: this is a foreground task + # note: this is a foreground task raise ServerExit @@ -677,7 +677,7 @@ class RestartTask(BaseTaskHandler): _taskWeight = 0.1 Foreground = True def handler(self, host): - #note: this is a foreground task + # note: this is a foreground task if host['id'] != self.session.host.getID(): raise koji.GenericError("Host mismatch") self.manager.restart_pending = True @@ -691,7 +691,7 @@ class RestartVerifyTask(BaseTaskHandler): _taskWeight = 0.1 Foreground = True def handler(self, task_id, host): - #note: this is a foreground task + # note: this is a foreground task tinfo = self.session.getTaskInfo(task_id) state = koji.TASK_STATES[tinfo['state']] if state != 'CLOSED': @@ -754,7 +754,7 @@ class RestartHostsTask(BaseTaskHandler): class DependantTask(BaseTaskHandler): Methods = ['dependantTask'] - #mostly just waiting on other tasks + # mostly just waiting on other tasks _taskWeight = 0.2 def handler(self, wait_list, task_list): diff --git a/koji/util.py b/koji/util.py index ec6c83d..350c03f 100644 --- a/koji/util.py +++ b/koji/util.py @@ -189,7 +189,7 @@ def dslice(dict_, keys, strict=True): ret = {} for key in keys: if strict or key in dict_: - #for strict we skip the has_key check and let the dict generate the KeyError + # for strict we skip the has_key check and let the dict generate the KeyError ret[key] = dict_[key] return ret @@ -639,13 +639,13 @@ def setup_rlimits(opts, logger=None): class adler32_constructor(object): - #mimicing the hashlib constructors + # mimicing the hashlib constructors def __init__(self, arg=''): if six.PY3 and isinstance(arg, str): arg = bytes(arg, 'utf-8') self._value = adler32(arg) & 0xffffffff - #the bitwise and works around a bug in some versions of python - #see: https://bugs.python.org/issue1202 + # the bitwise and works around a bug in some versions of python + # see: https://bugs.python.org/issue1202 def update(self, arg): if six.PY3 and isinstance(arg, str): diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index 1a3c0db..40fb536 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -118,9 +118,10 @@ class RunRootTask(koji.tasks.BaseTaskHandler): if weight is not None: weight = max(weight, 0.5) self.session.host.setTaskWeight(self.id, weight) - #noarch is funny + + # noarch is funny if arch == "noarch": - #we need a buildroot arch. Pick one that: + # we need a buildroot arch. Pick one that: # a) this host can handle # b) the build tag can support # c) is canonical @@ -130,16 +131,16 @@ class RunRootTask(koji.tasks.BaseTaskHandler): tag_arches = self.session.getBuildConfig(root)['arches'] if not tag_arches: raise koji.BuildError("No arch list for tag: %s" % root) - #index canonical host arches + # index canonical host arches host_arches = set([koji.canonArch(a) for a in host_arches.split()]) - #pick the first suitable match from tag's archlist + # pick the first suitable match from tag's archlist for br_arch in tag_arches.split(): br_arch = koji.canonArch(br_arch) if br_arch in host_arches: - #we're done + # we're done break else: - #no overlap + # no overlap raise koji.BuildError("host does not match tag arches: %s (%s)" % (root, tag_arches)) else: br_arch = arch @@ -152,7 +153,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): else: repo_info = self.session.getRepo(root) if not repo_info: - #wait for it + # wait for it task_id = self.session.host.subtask(method='waitrepo', arglist=[root, None, None], parent=self.id) @@ -163,13 +164,13 @@ class RunRootTask(koji.tasks.BaseTaskHandler): broot.workdir = self.workdir broot.init() rootdir = broot.rootdir() - #workaround for rpm oddness + # workaround for rpm oddness os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir) - #update buildroot state (so that updateBuildRootList() will work) + # update buildroot state (so that updateBuildRootList() will work) self.session.host.setBuildRootState(broot.id, 'BUILDING') try: if packages: - #pkglog = '%s/%s' % (broot.resultdir(), 'packages.log') + # pkglog = '%s/%s' % (broot.resultdir(), 'packages.log') pkgcmd = ['--install'] + packages status = broot.mock(pkgcmd) self.session.host.updateBuildRootList(broot.id, broot.getPackageList()) @@ -179,9 +180,9 @@ class RunRootTask(koji.tasks.BaseTaskHandler): if isinstance(command, str): cmdstr = command else: - #we were passed an arglist - #we still have to run this through the shell (for redirection) - #but we can preserve the list structure precisely with careful escaping + # we were passed an arglist + # we still have to run this through the shell (for redirection) + # but we can preserve the list structure precisely with careful escaping cmdstr = ' '.join(["'%s'" % arg.replace("'", r"'\''") for arg in command]) # A nasty hack to put command output into its own file until mock can be # patched to do something more reasonable than stuff everything into build.log @@ -198,7 +199,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): elif new_chroot is False: # None -> no option added mock_cmd.append('--old-chroot') if skip_setarch: - #we can't really skip it, but we can set it to the current one instead of of the chroot one + # we can't really skip it, but we can set it to the current one instead of of the chroot one myarch = platform.uname()[5] mock_cmd.extend(['--arch', myarch]) mock_cmd.append('--') @@ -235,9 +236,9 @@ class RunRootTask(koji.tasks.BaseTaskHandler): if mount.startswith(safe_root): break else: - #no match + # no match raise koji.GenericError("read-write mount point is not safe: %s" % mount) - #normpath should have removed any .. dirs, but just in case... + # normpath should have removed any .. dirs, but just in case... if mount.find('/../') != -1: raise koji.GenericError("read-write mount point is not safe: %s" % mount) @@ -266,7 +267,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): else: opts = opts.split(',') if 'bind' in opts: - #make sure dir exists + # make sure dir exists if not os.path.isdir(dev): error = koji.GenericError("No such directory or mount: %s" % dev) break @@ -297,7 +298,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): with open(fn, 'r') as fslog: for line in fslog.readlines(): mounts.add(line.strip()) - #also, check /proc/mounts just in case + # also, check /proc/mounts just in case mounts |= set(scan_mounts(rootdir)) mounts = sorted(mounts) # deeper directories first diff --git a/plugins/hub/runroot_hub.py b/plugins/hub/runroot_hub.py index eacbd68..8dfda89 100644 --- a/plugins/hub/runroot_hub.py +++ b/plugins/hub/runroot_hub.py @@ -1,4 +1,4 @@ -#koji hub plugin +# koji hub plugin # There is a kojid plugin that goes with this hub plugin. The kojid builder # plugin has a config file. This hub plugin has no config file. @@ -15,7 +15,6 @@ import kojihub from koji.context import context from koji.plugin import export - __all__ = ('runroot',) @@ -41,11 +40,11 @@ def runroot(tagInfo, arch, command, channel=None, **opts): tag = kojihub.get_tag(tagInfo, strict=True) if arch == 'noarch': - #not all arches can generate a proper buildroot for all tags + # not all arches can generate a proper buildroot for all tags if not tag['arches']: raise koji.GenericError('no arches defined for tag %s' % tag['name']) - #get all known arches for the system + # get all known arches for the system fullarches = kojihub.get_all_arches() tagarches = tag['arches'].split() diff --git a/setup.py b/setup.py index bbb7bcc..a3443bc 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,9 @@ def get_install_requires(): 'requests', 'requests-kerberos', 'six', - #'libcomps', - #'rpm-py-installer', # it is optional feature - #'rpm', + # 'libcomps', + # 'rpm-py-installer', # it is optional feature + # 'rpm', ] if sys.version_info[0] < 3: # optional auth library for older hubs @@ -62,9 +62,9 @@ setup( 'koji_cli_plugins': 'plugins/cli', }, # doesn't make sense, as we have only example config - #data_files=[ - # ('/etc', ['cli/koji.conf']), - #], + # data_files=[ + # ('/etc', ['cli/koji.conf']), + # ], scripts=[ 'cli/koji', 'util/koji-gc', diff --git a/util/kojira b/util/kojira index 2336020..fdf492b 100755 --- a/util/kojira +++ b/util/kojira @@ -50,7 +50,7 @@ def getTag(session, tag, event=None): if (tag, event) in cache: ts, info = cache[(tag,event)] if now - ts < 600: - #use the cache + # use the cache return info info = session.getTag(tag, event=event) if info: @@ -83,7 +83,7 @@ class ManagedRepo(object): self.first_seen = time.time() if self.current: order = self.session.getFullInheritance(self.tag_id, event=self.event_id) - #order may contain same tag more than once + # order may contain same tag more than once tags = {self.tag_id : 1} for x in order: tags[x['parent_id']] = 1 @@ -156,13 +156,13 @@ class ManagedRepo(object): - timestamp really, really old """ timeout = 36000 - #XXX - config + # XXX - config if self.state != koji.REPO_INIT: return False age = time.time() - max(self.event_ts, self.first_seen) - #the first_seen timestamp is also factored in because a repo can be - #created from an older event and should not be expired based solely on - #that event's timestamp. + # the first_seen timestamp is also factored in because a repo can be + # created from an older event and should not be expired based solely on + # that event's timestamp. return age > timeout def tryDelete(self): @@ -177,8 +177,8 @@ class ManagedRepo(object): lifetime = self.options.deleted_repo_lifetime # (should really be called expired_repo_lifetime) try: - #also check dir age. We do this because a repo can be created from an older event - #and should not be removed based solely on that event's timestamp. + # also check dir age. We do this because a repo can be created from an older event + # and should not be removed based solely on that event's timestamp. mtime = os.stat(path).st_mtime except OSError as e: if e.errno == 2: @@ -200,7 +200,7 @@ class ManagedRepo(object): if self.state != koji.REPO_EXPIRED: raise koji.GenericError("Repo not expired") if self.session.repoDelete(self.repo_id) > 0: - #cannot delete, we are referenced by a buildroot + # cannot delete, we are referenced by a buildroot self.logger.debug("Cannot delete repo %s, still referenced" % self.repo_id) return False self.logger.info("Deleted repo %s" % self.repo_id) @@ -299,9 +299,9 @@ class RepoManager(object): (childpid, status) = os.waitpid(pid, os.WNOHANG) except OSError as e: if e.errno != errno.ECHILD: - #should not happen + # should not happen raise - #otherwise assume the process is gone + # otherwise assume the process is gone self.logger.info("%s: %s" % (prefix, e)) return True if childpid != 0: @@ -345,7 +345,7 @@ class RepoManager(object): repo_id = data['id'] repo = self.repos.get(repo_id) if repo: - #we're already tracking it + # we're already tracking it if repo.state != data['state']: self.logger.info('State changed for repo %s: %s -> %s' %(repo_id, koji.REPO_STATES[repo.state], koji.REPO_STATES[data['state']])) @@ -383,7 +383,7 @@ class RepoManager(object): repo.current = False if repo.expire_ts is None: repo.expire_ts = time.time() - #also no point in further checking + # also no point in further checking continue to_check.append(repo) if self.logger.isEnabledFor(logging.DEBUG): @@ -441,7 +441,7 @@ class RepoManager(object): Also, warn about any oddities""" if self.delete_pids: - #skip + # skip return if not os.path.exists(topdir): self.logger.debug("%s doesn't exist, skipping", topdir) @@ -466,14 +466,14 @@ class RepoManager(object): self.logger.debug("%s/%s not an int, skipping", tagdir, repo_id) continue if repo_id in self.repos: - #we're already managing it, no need to deal with it here + # we're already managing it, no need to deal with it here continue repodir = "%s/%s" % (tagdir, repo_id) try: # lstat because it could be link to another volume dirstat = os.lstat(repodir) except OSError: - #just in case something deletes the repo out from under us + # just in case something deletes the repo out from under us self.logger.debug("%s deleted already?!", repodir) continue symlink = False @@ -513,18 +513,18 @@ class RepoManager(object): stats = self.tag_use_stats.get(tag_id) now = time.time() if stats and now - stats['ts'] < 3600: - #use the cache + # use the cache return stats data = self.session.listBuildroots(tagID=tag_id, queryOpts={'order': '-create_event_id', 'limit' : 100}) - #XXX magic number (limit) + # XXX magic number (limit) if data: tag_name = data[0]['tag_name'] else: tag_name = "#%i" % tag_id stats = {'data': data, 'ts': now, 'tag_name': tag_name} recent = [x for x in data if now - x['create_ts'] < 3600 * 24] - #XXX magic number + # XXX magic number stats ['n_recent'] = len(recent) self.tag_use_stats[tag_id] = stats self.logger.debug("tag %s recent use count: %i" % (tag_name, len(recent))) @@ -593,7 +593,7 @@ class RepoManager(object): if n_deletes >= self.options.delete_batch_size: break if repo.expired(): - #try to delete + # try to delete if repo.tryDelete(): n_deletes += 1 del self.repos[repo.repo_id] @@ -652,7 +652,7 @@ class RepoManager(object): t['build_tag'] for t in self.session.getBuildTargets() if not koji.util.multi_fnmatch(t['build_tag_name'], ignore) ]) - #index repos by tag + # index repos by tag tag_repos = {} for repo in to_list(self.repos.values()): tag_repos.setdefault(repo.tag_id, []).append(repo) @@ -931,7 +931,7 @@ def get_options(): 'repo_tasks_limit' : 10, 'delete_batch_size' : 3, 'deleted_repo_lifetime': 7*24*3600, - #XXX should really be called expired_repo_lifetime + # XXX should really be called expired_repo_lifetime 'dist_repo_lifetime': 7*24*3600, 'recent_tasks_lifetime': 600, 'sleeptime' : 15, @@ -1003,7 +1003,7 @@ if __name__ == "__main__": sys.stderr.write("Cannot write to logfile: %s\n" % options.logfile) sys.exit(1) koji.add_file_logger("koji", options.logfile) - #note we're setting logging for koji.* + # note we're setting logging for koji.* logger = logging.getLogger("koji") if options.debug: logger.setLevel(logging.DEBUG) @@ -1024,7 +1024,7 @@ if __name__ == "__main__": session.login() elif koji.krbV and options.principal and options.keytab: session.krb_login(options.principal, options.keytab, options.ccache) - #get an exclusive session + # get an exclusive session try: session.exclusiveSession(force=options.force_lock) except koji.AuthLockError: diff --git a/vm/fix_kojikamid.sh b/vm/fix_kojikamid.sh index 5b0f5c1..f0063a7 100755 --- a/vm/fix_kojikamid.sh +++ b/vm/fix_kojikamid.sh @@ -1,10 +1,10 @@ #!/bin/bash -awk '/^## INSERT kojikamid dup/ {exit} {print $0}' kojikamid.py +awk '/^# INSERT kojikamid dup #/ {exit} {print $0}' kojikamid.py for fn in ../koji/__init__.py ../koji/daemon.py do - awk '/^## END kojikamid dup/ {p=0} p {print $0} /^## BEGIN kojikamid dup/ {p=1}' $fn + awk '/^# END kojikamid dup #/ {p=0} p {print $0} /^# BEGIN kojikamid dup #/ {p=1}' $fn done -awk 'p {print $0} /^## INSERT kojikamid dup/ {p=1}' kojikamid.py +awk 'p {print $0} /^# INSERT kojikamid dup #/ {p=1}' kojikamid.py diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 311b2e6..0911a1a 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -54,12 +54,12 @@ MANAGER_PORT = 7000 KOJIKAMID = True -## INSERT kojikamid dup +# INSERT kojikamid dup # class fakemodule(object): pass -#make parts of the above insert accessible as koji.X +# make parts of the above insert accessible as koji.X koji = fakemodule() koji.GenericError = GenericError # noqa: F821 koji.BuildError = BuildError # noqa: F821 @@ -68,7 +68,7 @@ def encode_int(n): """If n is too large for a 32bit signed, convert it to a string""" if n <= 2147483647: return n - #else + # else return str(n) class WindowsBuild(object): diff --git a/vm/kojivmd b/vm/kojivmd index c0fa3ff..983885f 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -101,7 +101,7 @@ def get_options(): if args: parser.error("incorrect number of arguments") - #not reached + # not reached assert False # pragma: no cover # load local config @@ -176,7 +176,7 @@ def get_options(): if os.path.exists(fn): setattr(options, name, fn) - #make sure workdir exists + # make sure workdir exists if not os.path.exists(options.workdir): koji.ensuredir(options.workdir) @@ -198,7 +198,7 @@ def main(options, session): tm = VMTaskManager(options, session) tm.findHandlers(globals()) if options.plugin: - #load plugins + # load plugins pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':')) for name in options.plugin: logger.info('Loading plugin: %s', name) @@ -1084,7 +1084,7 @@ class VMTaskManager(TaskManager): if __name__ == "__main__": koji.add_file_logger("koji", "/var/log/kojivmd.log") - #note we're setting logging params for all of koji* + # note we're setting logging params for all of koji* options = get_options() if options.debug: logging.getLogger("koji").setLevel(logging.DEBUG) @@ -1097,7 +1097,7 @@ if __name__ == "__main__": if options.admin_emails: koji.add_mail_logger("koji", options.admin_emails) - #start a session and login + # start a session and login session_opts = koji.grab_session_options(options) session = koji.ClientSession(options.server, session_opts) if options.cert and os.path.isfile(options.cert): @@ -1131,14 +1131,14 @@ if __name__ == "__main__": quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1]) else: quit("No username/password supplied and Kerberos missing or not configured") - #make session exclusive + # make session exclusive try: session.exclusiveSession(force=options.force_lock) except koji.AuthLockError: quit("Error: Unable to get lock. Trying using --force-lock") if not session.logged_in: quit("Error: Unknown login error") - #make sure it works + # make sure it works try: ret = session.echo("OK") except requests.exceptions.ConnectionError: @@ -1148,7 +1148,7 @@ if __name__ == "__main__": # run main if options.daemon: - #detach + # detach koji.daemonize() main(options, session) elif not options.skip_main: diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 07f911e..19414bf 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -45,7 +45,7 @@ from kojiweb.util import _genHTML, _getValidTokens, _initValues # Convenience definition of a commonly-used sort function _sortbyname = lambda x: x['name'] -#loggers +# loggers authlogger = logging.getLogger('koji.auth') def _setUserCookie(environ, user): @@ -790,7 +790,7 @@ def getfile(environ, taskID, name, volume='DEFAULT', offset=None, size=None): if size > (file_size - offset): size = file_size - offset - #environ['koji.headers'].append(['Content-Length', str(size)]) + # environ['koji.headers'].append(['Content-Length', str(size)]) return _chunk_file(server, environ, taskID, name, offset, size, volume) diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 7f15a55..50e3450 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -44,7 +44,7 @@ class URLNotFound(ServerError): class Dispatcher(object): def __init__(self): - #we can't do much setup until we get a request + # we can't do much setup until we get a request self.firstcall = True self.options = {} self.startup_error = None @@ -66,7 +66,7 @@ class Dispatcher(object): self.logger = logging.getLogger("koji.web") cfgmap = [ - #option, type, default + # option, type, default ['SiteName', 'string', None], ['KojiHubURL', 'string', 'http://localhost/kojihub'], ['KojiFilesURL', 'string', 'http://localhost/kojifiles'], @@ -156,7 +156,7 @@ class Dispatcher(object): def setup_logging2(self, environ): """Adjust logging based on configuration options""" opts = self.options - #determine log level + # determine log level level = opts['LogLevel'] valid_levels = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL') # the config value can be a single level name or a series of @@ -172,7 +172,7 @@ class Dispatcher(object): default = level if level not in valid_levels: raise koji.GenericError("Invalid log level: %s" % level) - #all our loggers start with koji + # all our loggers start with koji if name == '': name = 'koji' default = level @@ -187,7 +187,7 @@ class Dispatcher(object): if opts.get('KojiDebug'): logger.setLevel(logging.DEBUG) elif default is None: - #LogLevel did not configure a default level + # LogLevel did not configure a default level logger.setLevel(logging.WARNING) self.formatter = HubFormatter(opts['LogFormat']) self.formatter.environ = environ @@ -213,7 +213,7 @@ class Dispatcher(object): def prep_handler(self, environ): path_info = environ['PATH_INFO'] if not path_info: - #empty path info (no trailing slash) breaks our relative urls + # empty path info (no trailing slash) breaks our relative urls environ['koji.redirect'] = environ['REQUEST_URI'] + '/' raise ServerRedirect elif path_info == '/': @@ -225,7 +225,7 @@ class Dispatcher(object): func = self.handler_index.get(method) if not func: raise URLNotFound - #parse form args + # parse form args data = {} fs = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ.copy(), keep_blank_values=True) for field in fs.list: @@ -245,7 +245,7 @@ class Dispatcher(object): if not varkw: # remove any unexpected args data = dslice(data, args, strict=False) - #TODO (warning in header or something?) + # TODO (warning in header or something?) return func, data @@ -318,7 +318,7 @@ class Dispatcher(object): except (NameError, AttributeError): tb_str = ''.join(traceback.format_exception(*sys.exc_info())) self.logger.error(tb_str) - #fallback to simple error page + # fallback to simple error page return self.simple_error_page(message, err=tb_short) values = _initValues(environ, *desc) values['etype'] = etype diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index a318126..f7709d8 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -26,7 +26,7 @@ import hashlib import os import ssl import stat -#a bunch of exception classes that explainError needs +# a bunch of exception classes that explainError needs from socket import error as socket_error from xml.parsers.expat import ExpatError From a0a9dd74caad2796a79ec3a11c92374e2de6c236 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:35:08 +0000 Subject: [PATCH 3/23] flake8: util/koji-* were ignored --- diff --git a/.flake8 b/.flake8 index 0a8cb0c..7e4b2a4 100644 --- a/.flake8 +++ b/.flake8 @@ -1,19 +1,23 @@ [flake8] select = I,C,F -exclude = .git, - __pycache__, - tests, - docs, - ./koji-* -filename = *.py - ./cli/koji - ./builder/kojid - ./builder/mergerepos - ./hub/rpmdiff - ./util/kojira - ./util/koji-gc - ./util/koji-shadow - ./util/koji-sweep-db - ./vm/kojivmd +exclude = + .git, + __pycache__, + tests, + docs, + ./koji-*/* + +filename = + *.py, + ./cli/koji, + ./builder/kojid, + ./builder/mergerepos, + ./hub/rpmdiff, + ./util/kojira, + ./util/koji-gc, + ./util/koji-shadow, + ./util/koji-sweep-db, + ./vm/kojivmd + application_import_names = koji,koji_cli,kojihub,kojiweb,__main__ import_order_style = pep8 diff --git a/util/koji-gc b/util/koji-gc index a94d02f..9d75b4c 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -7,8 +7,9 @@ # Mike McLean from __future__ import absolute_import -import fcntl + import datetime +import fcntl import fnmatch import optparse import os From 49504073b18696210bffa846803bfad5900d0382 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Mar 03 2020 13:35:08 +0000 Subject: [PATCH 4/23] refine import style --- diff --git a/builder/kojid b/builder/kojid index cf4d060..a775b5d 100755 --- a/builder/kojid +++ b/builder/kojid @@ -60,8 +60,12 @@ import koji.rpmdiff import koji.tasks import koji.util from koji.daemon import SCM, TaskManager, incremental_upload, log_output -from koji.tasks import (BaseTaskHandler, MultiPlatformTask, ServerExit, - ServerRestart) +from koji.tasks import ( + BaseTaskHandler, + MultiPlatformTask, + ServerExit, + ServerRestart +) from koji.util import dslice, dslice_ex, isSuccess, parseStatus, to_list try: diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index f23c8a1..f41b573 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -23,12 +23,27 @@ from six.moves import filter, map, range, zip import koji from koji.util import base64encode, to_list -from koji_cli.lib import (_, _list_tasks, _progress_callback, _running_in_bg, - activate_session, arg_filter, download_file, error, - format_inheritance_flags, get_usage_str, greetings, - linked_upload, list_task_output_all_volumes, - print_task_headers, print_task_recurse, unique_path, - warn, watch_logs, watch_tasks) +from koji_cli.lib import ( + _, + _list_tasks, + _progress_callback, + _running_in_bg, + activate_session, + arg_filter, + download_file, + error, + format_inheritance_flags, + get_usage_str, + greetings, + linked_upload, + list_task_output_all_volumes, + print_task_headers, + print_task_recurse, + unique_path, + warn, + watch_logs, + watch_tasks +) try: import libcomps diff --git a/hub/kojihub.py b/hub/kojihub.py index b9cba9f..6699359 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -60,8 +60,16 @@ import koji.rpmdiff import koji.tasks import koji.xmlrpcplus from koji.context import context -from koji.util import (base64encode, decode_bytes, dslice, joinpath, - move_and_symlink, multi_fnmatch, safer_move, to_list) +from koji.util import ( + base64encode, + decode_bytes, + dslice, + joinpath, + move_and_symlink, + multi_fnmatch, + safer_move, + to_list +) try: # py 3.6+ diff --git a/koji/daemon.py b/koji/daemon.py index 9f5fb56..684864c 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -40,8 +40,13 @@ import koji import koji.tasks import koji.xmlrpcplus from koji.tasks import safe_rmtree -from koji.util import (adler32_constructor, base64encode, dslice, parseStatus, - to_list) +from koji.util import ( + adler32_constructor, + base64encode, + dslice, + parseStatus, + to_list +) def incremental_upload(session, fname, fd, path, retries=5, logger=None): diff --git a/plugins/cli/runroot.py b/plugins/cli/runroot.py index ba448c4..e1258a7 100644 --- a/plugins/cli/runroot.py +++ b/plugins/cli/runroot.py @@ -6,8 +6,13 @@ from optparse import OptionParser import koji from koji.plugin import export_cli -from koji_cli.lib import (_, activate_session, bytes_to_stdout, - list_task_output_all_volumes, watch_tasks) +from koji_cli.lib import ( + _, + activate_session, + bytes_to_stdout, + list_task_output_all_volumes, + watch_tasks +) @export_cli diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py index 341b2bd..1a25a80 100644 --- a/plugins/cli/sidetag_cli.py +++ b/plugins/cli/sidetag_cli.py @@ -9,8 +9,8 @@ from argparse import ArgumentParser import koji from koji.plugin import export_cli -from koji_cli.lib import _, activate_session from koji_cli.commands import anon_handle_wait_repo +from koji_cli.lib import _, activate_session @export_cli diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 2f9bd45..92af160 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -3,27 +3,26 @@ # SPDX-License-Identifier: GPL-2.0-or-later import sys -from koji.context import context -from koji.plugin import export, callback import koji - -CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" -CONFIG = None - +from koji.context import context +from koji.plugin import callback, export sys.path.insert(0, "/usr/share/koji-hub/") from kojihub import ( + QueryProcessor, + _create_build_target, + _create_tag, + _delete_build_target, + _delete_tag, assert_policy, + get_build_target, get_tag, get_user, - get_build_target, - _create_tag, - _create_build_target, - _delete_tag, - _delete_build_target, - QueryProcessor, - nextval, + nextval ) +CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" +CONFIG = None + @export def createSideTag(basetag): diff --git a/vm/kojivmd b/vm/kojivmd index 983885f..3564856 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -48,8 +48,14 @@ import koji import koji.util from koji.daemon import SCM, TaskManager # TaskHandlers are required to be imported, do not remove them -from koji.tasks import (BaseTaskHandler, MultiPlatformTask, RestartTask, # noqa: F401 - RestartVerifyTask, ServerExit, ServerRestart) +from koji.tasks import ( # noqa: F401 + BaseTaskHandler, + MultiPlatformTask, + RestartTask, + RestartVerifyTask, + ServerExit, + ServerRestart +) try: import krbV From 0a0ee577ec5d7958315155a72f3ab353a4cd6793 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:35:08 +0000 Subject: [PATCH 5/23] flake8: apply F rules for koji-shadow --- diff --git a/util/koji-shadow b/util/koji-shadow index ea3d56a..7ac338e 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -182,23 +182,6 @@ def get_options(): else: log(config.get(*alias)) setattr(defaults, name, config.get(*alias)) - #config file options without a cmdline equivalent - otheropts = [ - #name, type, default - ['keytab', None, 'string'], - ['principal', None, 'string'], - ['runas', None, 'string'], - ['user', None, 'string'], - ['password', None, 'string'], - ['noauth', None, 'boolean'], - ['server', None, 'string'], - ['remote', None, 'string'], - ['max_jobs', None, 'int'], - ['serverca', None, 'string'], - ['auth_cert', None, 'string'], - ['arches', None, 'string'], - ] - #parse again with updated defaults (options, args) = parser.parse_args(values=defaults) @@ -921,11 +904,6 @@ class BuildTracker(object): self.tagSuccessful(build.nvr, tag) return True - def scan(self): - """Scan based on config file""" - to_scan = [] - alltags = remote.listTags() - def rebuild(self, build): """Rebuild a remote build using closest possible buildroot""" #first check that we can From 450f9249cdd089eea55e72ad2736b8c05d2802ce Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:35:08 +0000 Subject: [PATCH 6/23] flake8: apply E265 for util/koji-* --- diff --git a/.flake8 b/.flake8 index 7e4b2a4..27b4d7f 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -select = I,C,F +select = I,C,F,E265 exclude = .git, __pycache__, diff --git a/util/koji-gc b/util/koji-gc index 9d75b4c..48f1dd7 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -123,7 +123,7 @@ def get_options(): "recommended.")) parser.add_option("--exit-on-lock", action="store_true", help=_("quit if --lock-file exists, don't wait")) - #parse once to get the config file + # parse once to get the config file (options, args) = parser.parse_args() defaults = parser.get_default_values() @@ -177,11 +177,11 @@ def get_options(): setattr(defaults, name, config.getboolean(*alias)) else: setattr(defaults, name, config.get(*alias)) - #parse again with defaults + # parse again with defaults (options, args) = parser.parse_args(values=defaults) options.config = config - #figure out actions + # figure out actions actions = ('prune', 'trash', 'delete', 'salvage') if options.action: options.action = options.action.lower().replace(',',' ').split() @@ -191,13 +191,13 @@ def get_options(): else: options.action = ('delete', 'prune', 'trash') - #split patterns for unprotected keys + # split patterns for unprotected keys if options.unprotected_keys: options.unprotected_key_patterns = options.unprotected_keys.replace(',',' ').split() else: options.unprotected_key_patterns = [] - #parse key aliases + # parse key aliases options.key_aliases = {} try: if config.has_option('main', 'key_aliases'): @@ -210,7 +210,7 @@ def get_options(): print(e) parser.error(_("Invalid key alias data in config: %s") % config.get('main','key_aliases')) - #parse time intervals + # parse time intervals for key in ('delay', 'grace_period'): try: value = getattr(options, key) @@ -252,10 +252,10 @@ def check_tag(name): for pattern in options.tag_filter: if fnmatch.fnmatch(name, pattern): return True - #doesn't match any pattern in filter + # doesn't match any pattern in filter return False else: - #not ignored and no filter specified + # not ignored and no filter specified return True def check_package(name): @@ -267,10 +267,10 @@ def check_package(name): for pattern in options.pkg_filter: if fnmatch.fnmatch(name, pattern): return True - #doesn't match any pattern in filter + # doesn't match any pattern in filter return False else: - #no filter specified + # no filter specified return True time_units = { @@ -281,7 +281,7 @@ time_units = { 'week' : 604800, } time_unit_aliases = [ - #[unit, alias, alias, ...] + # [unit, alias, alias, ...] ['week', 'weeks', 'wk', 'wks'], ['hour', 'hours', 'hr', 'hrs'], ['day', 'days'], @@ -308,7 +308,7 @@ def parse_duration(str): n = parse_num(x) if n is not None: continue - #perhaps the unit is appended w/o a space + # perhaps the unit is appended w/o a space for names in time_unit_aliases: for name in names: if x.endswith(name): @@ -371,13 +371,13 @@ def activate_session(session): """Test and login the session is applicable""" global options if options.noauth: - #skip authentication + # skip authentication pass elif options.cert is not None and os.path.isfile(options.cert): # authenticate using SSL client cert session.ssl_login(options.cert, None, options.serverca, proxyuser=options.runas) elif options.user: - #authenticate using user/password + # authenticate using user/password session.login() elif has_krb_creds() or (options.keytab and options.principal): try: @@ -452,7 +452,7 @@ def handle_trash(): print("...got %i builds" % len(untagged)) min_age = options.delay trashcan_tag = options.trashcan_tag - #Step 1: place unreferenced builds into trashcan + # Step 1: place unreferenced builds into trashcan i = 0 N = len(untagged) to_trash = [] @@ -477,7 +477,7 @@ def handle_trash(): for binfo, [refs] in six.moves.zip(continuing, mcall.call_all()): i += 1 nvr = binfo['nvr'] - #XXX - this is more data than we need + # XXX - this is more data than we need # also, this call takes waaaay longer than it should if refs.get('tags'): # must have been tagged just now @@ -486,12 +486,12 @@ def handle_trash(): if refs.get('rpms'): if options.debug: print("[%i/%i] Build has %i rpm references: %s" % (i, N, len(refs['rpms']), nvr)) - #pprint.pprint(refs['rpms']) + # pprint.pprint(refs['rpms']) continue if refs.get('archives'): if options.debug: print("[%i/%i] Build has %i archive references: %s" % (i, N, len(refs['archives']), nvr)) - #pprint.pprint(refs['archives']) + # pprint.pprint(refs['archives']) continue if refs.get('component_of'): if options.debug: @@ -499,22 +499,22 @@ def handle_trash(): continue ts = refs['last_used'] if ts: - #work around server bug + # work around server bug if isinstance(ts, list): ts = ts[0] - #XXX - should really check time server side + # XXX - should really check time server side if options.debug: print("[%i/%i] Build has been used in a buildroot: %s" % (i, N, nvr)) print("Last_used: %s" % datetime.datetime.fromtimestamp(ts).isoformat()) age = time.time() - ts if age < min_age: continue - #see how long build has been untagged + # see how long build has been untagged history = session.queryHistory(build=binfo['id'])['tag_listing'] age = None binfo2 = None if not history: - #never tagged, we'll have to use the build create time + # never tagged, we'll have to use the build create time binfo2 = session.getBuild(binfo['id']) ts = binfo2.get('creation_ts') if ts is None: @@ -531,7 +531,7 @@ def handle_trash(): history = [(h['revoke_event'],h) for h in history] last = max(history)[1] if not last['revoke_event']: - #this might happen if the build was tagged just now + # this might happen if the build was tagged just now print("[%i/%i] Warning: build not untagged: %s" % (i, N, nvr)) continue age = time.time() - last['revoke_ts'] @@ -539,7 +539,7 @@ def handle_trash(): if options.debug: print("[%i/%i] Build untagged only recently: %s" % (i, N, nvr)) continue - #check build signatures + # check build signatures keys = get_build_sigs(binfo['id'], cache=True) if keys and options.debug: print("Build: %s, Keys: %s" % (nvr, keys)) @@ -547,14 +547,14 @@ def handle_trash(): print("Skipping build %s. Keys: %s" % (nvr, keys)) continue - #ok, go ahead add it to the list + # ok, go ahead add it to the list if binfo2 is None: binfo2 = session.getBuild(binfo['id']) print("[%i/%i] Adding build to trash list: %s" % (i, N, nvr)) to_trash.append(binfo2) - #process to_trash - #group by owner so we can reduce the number of notices + # process to_trash + # group by owner so we can reduce the number of notices by_owner = {} for binfo in to_trash: by_owner.setdefault(binfo['owner_name'], []).append(binfo) @@ -571,14 +571,14 @@ def handle_trash(): else: if options.debug: print("Moving to trashcan: %s" % nvr) - #figure out package owner + # figure out package owner count = {} for pkg in session.listPackages(pkgID=binfo['name']): count.setdefault(pkg['owner_id'], 0) count[pkg['owner_id']] += 1 if not count: print("Warning: no owner for %s, using build owner" % nvr) - #best we can do currently + # best we can do currently owner = binfo['owner_id'] else: owner = max([(n, k) for k, n in six.iteritems(count)])[1] @@ -597,7 +597,7 @@ def protected_sig(keys): if not key: continue if not sigmatch(key, options.unprotected_key_patterns): - #this key is protected + # this key is protected return True return False @@ -633,7 +633,7 @@ def handle_delete(just_salvage=False): trash = [(b['nvr'], b) for b in session.listTagged(trashcan_tag)] trash.sort() print("...got %i builds" % len(trash)) - #XXX - it would be better if there were more appropriate server calls for this + # XXX - it would be better if there were more appropriate server calls for this grace_period = options.grace_period import time @@ -685,7 +685,7 @@ def handle_delete(just_salvage=False): for (nvr, binfo), [history] in zip(trash, mcall.call_all()): current = [x for x in history if x['active']] if not current: - #untagged just now? + # untagged just now? print("Warning: history missing for %s" % nvr) pprint.pprint(binfo) pprint.pprint(history) @@ -712,7 +712,7 @@ def handle_delete(just_salvage=False): for binfo, result in six.moves.zip(continuing, mcall.call_all()): if isinstance(result, dict): print("Warning: deletion failed: %s" % result['faultString']) - #TODO - log details for delete failures + # TODO - log details for delete failures class TagPruneTest(koji.policy.MatchTest): @@ -814,7 +814,7 @@ def get_build_sigs(build, cache=False): ret = build_sig_cache[build] = [] return ret else: - #TODO - multicall helps, but it might be good to have a more robust server-side call + # TODO - multicall helps, but it might be good to have a more robust server-side call session.multicall = True for rpminfo in rpms: session.queryRPMSigs(rpm_id=rpminfo['id']) @@ -830,18 +830,18 @@ def handle_prune(): If purge is True, will also attempt to delete the pruned builds afterwards """ - #read policy + # read policy if not options.config or not options.config.has_option('prune', 'policy'): print("Skipping prune step. No policies available.") return - #policies = read_policies(options.policy_file) + # policies = read_policies(options.policy_file) policies = scan_policies(options.config.get('prune', 'policy')) for action in policies.all_actions(): if action not in ("keep", "untag", "skip"): raise Exception("Invalid action: %s" % action) if options.debug: pprint.pprint(policies.ruleset) - #get tags + # get tags tags = session.listTags(perms=False, queryOpts={'order': 'name'}) untagged = {} build_ids = {} @@ -852,7 +852,7 @@ def handle_prune(): print("Skipping trashcan tag: %s" % tagname) continue if not check_tag(tagname): - #if options.debug: + # if options.debug: # print("skipping tag due to filter: %s" % tagname) continue bypass = False @@ -870,7 +870,7 @@ def handle_prune(): continue if options.debug: print("Pruning tag: %s" % tagname) - #get builds + # get builds history = session.queryHistory(tag=tagname, active=True)['tag_listing'] if not history: if options.debug: @@ -886,13 +886,13 @@ def handle_prune(): pkgs.sort() for pkg in pkgs: if not check_package(pkg): - #if options.debug: + # if options.debug: # print("skipping package due to filter: %s" % pkg) continue if options.debug: print(pkg) hist = pkghist[pkg] - #these are the *active* history entries for tag/pkg + # these are the *active* history entries for tag/pkg skipped = 0 for order, entry in enumerate(hist): # get sig data @@ -937,19 +937,19 @@ def handle_prune(): build_id = build_ids[nvr] tags = [t['name'] for t in session.listTags(build_id, perms=False)] if options.test: - #filted out the tags we would have dropped above + # filted out the tags we would have dropped above tags = [t for t in tags if t not in untagged[nvr]] if tags: - #still tagged somewhere + # still tagged somewhere print("Skipping %s, still tagged: %s" % (nvr, tags)) continue - #check cached sigs first to save a little time + # check cached sigs first to save a little time if build_id in build_sig_cache: keys = build_sig_cache[build_id] if protected_sig(keys): print("Skipping %s, signatures: %s" % (nvr, keys)) continue - #recheck signatures in case build was signed during run + # recheck signatures in case build was signed during run keys = get_build_sigs(build_id, cache=False) if protected_sig(keys): print("Skipping %s, signatures: %s" % (nvr, keys)) @@ -963,7 +963,7 @@ def handle_prune(): session.deleteBuild(build_id, strict=False) except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e: print("Warning: deletion failed: %s" % e) - #server issue + # server issue pass if __name__ == "__main__": @@ -1011,7 +1011,7 @@ if __name__ == "__main__": pass except SystemExit: rv = 1 - #except: + # except: # if options.debug: # raise # else: diff --git a/util/koji-shadow b/util/koji-shadow index 7ac338e..183c691 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -156,14 +156,14 @@ def get_options(): parser.add_option("--priority", type="int", default=5, help=_("priority to set for submitted builds")) - #parse once to get the config file + # parse once to get the config file (options, args) = parser.parse_args() defaults = parser.get_default_values() cf = getattr(options, 'config_file', '/etc/koji-shadow/koji-shadow.conf') config = koji.read_config_files(cf) - #allow config file to update defaults + # allow config file to update defaults for opt in parser.option_list: if not opt.dest: continue @@ -183,7 +183,7 @@ def get_options(): log(config.get(*alias)) setattr(defaults, name, config.get(*alias)) - #parse again with updated defaults + # parse again with updated defaults (options, args) = parser.parse_args(values=defaults) options.config = config @@ -197,7 +197,7 @@ time_units = { 'week' : 604800, } time_unit_aliases = [ - #[unit, alias, alias, ...] + # [unit, alias, alias, ...] ['week', 'weeks', 'wk', 'wks'], ['hour', 'hours', 'hr', 'hrs'], ['day', 'days'], @@ -224,7 +224,7 @@ def parse_duration(str): n = parse_num(x) if n is not None: continue - #perhaps the unit is appended w/o a space + # perhaps the unit is appended w/o a space for names in time_unit_aliases: for name in names: if x.endswith(name): @@ -278,7 +278,7 @@ def activate_session(session): global options if options.noauth: - #skip authentication + # skip authentication pass elif options.auth_cert and options.serverca: # convert to absolute paths @@ -289,7 +289,7 @@ def activate_session(session): # authenticate using SSL client cert session.ssl_login(cert=options.auth_cert, serverca=options.serverca, proxyuser=options.runas) elif options.user: - #authenticate using user/password + # authenticate using user/password session.login() elif krbV: try: @@ -347,9 +347,9 @@ class TrackedBuild(object): self.order = 0 self.substitute = None if child is not None: - #children tracks the builds that were built using this one + # children tracks the builds that were built using this one self.children[child] = 1 - #see if we have it + # see if we have it self.rebuilt = False self.updateState() if self.state == 'missing': @@ -374,7 +374,7 @@ class TrackedBuild(object): self.rebuilt = True return elif state in ('FAILED', 'CANCELED'): - #treat these as having no build + # treat these as having no build pass elif state == 'BUILDING' and ours['task_id']: self.setState("pending") @@ -392,14 +392,14 @@ class TrackedBuild(object): noarch = False for rpminfo in self.rpms: if rpminfo['arch'] == 'noarch': - #note that we've seen a noarch rpm + # note that we've seen a noarch rpm noarch = True elif rpminfo['arch'] != 'src': return False return noarch def setState(self, state): - #log("%s -> %s" % (self.nvr, state)) + # log("%s -> %s" % (self.nvr, state)) if state == self.state: return if self.state is not None and self.tracker: @@ -411,11 +411,11 @@ class TrackedBuild(object): def getSource(self): """Get source from remote""" if options.remote_topurl and self.srpm: - #download srpm from remote + # download srpm from remote pathinfo = koji.PathInfo(options.remote_topurl) url = "%s/%s" % (pathinfo.build(self.info), pathinfo.rpm(self.srpm)) log("Downloading %s" % url) - #XXX - this is not really the right place for this + # XXX - this is not really the right place for this fsrc = urllib2.urlopen(url) fn = "%s/%s.src.rpm" % (options.workpath, self.nvr) koji.ensuredir(os.path.dirname(fn)) @@ -427,7 +427,7 @@ class TrackedBuild(object): session.uploadWrapper(fn, serverdir, blocksize=65536) src = "%s/%s" % (serverdir, os.path.basename(fn)) return src - #otherwise use SCM url + # otherwise use SCM url task_id = self.info['task_id'] if task_id: tinfo = remote.getTaskInfo(task_id) @@ -435,12 +435,12 @@ class TrackedBuild(object): try: request = remote.getTaskRequest(task_id) src = request[0] - #XXX - Move SCM class out of kojid and use it to check for scm url + # XXX - Move SCM class out of kojid and use it to check for scm url if src.startswith('cvs:'): return src except: pass - #otherwise fail + # otherwise fail return None def addChild(self, child): @@ -485,18 +485,18 @@ class TrackedBuild(object): if br_id in seen: continue seen[br_id] = 1 - #br_info = remote.getBuildroot(br_id, strict=True) + # br_info = remote.getBuildroot(br_id, strict=True) remote.getBuildroot(br_id, strict=True) unpack.append(('br_info', br_id)) - #tags.setdefault(br_info['tag_name'], 0) - #tags[br_info['tag_name']] += 1 - #print(".") + # tags.setdefault(br_info['tag_name'], 0) + # tags[br_info['tag_name']] += 1 + # print(".") remote.listRPMs(componentBuildrootID=br_id) unpack.append(('rpmlist', br_id)) - #for rinfo in remote.listRPMs(componentBuildrootID=br_id): - # builds[rinfo['build_id']] = 1 - # if not rinfo['is_update']: - # bases.setdefault(rinfo['name'], {})[br_id] = 1 + # for rinfo in remote.listRPMs(componentBuildrootID=br_id): + # builds[rinfo['build_id']] = 1 + # if not rinfo['is_update']: + # bases.setdefault(rinfo['name'], {})[br_id] = 1 for (dtype, br_id), data in zip(unpack, remote.multiCall()): if dtype == 'br_info': [br_info] = data @@ -516,13 +516,13 @@ class TrackedBuild(object): # repo and others the new one. base = [] for name, brlist in six.iteritems(bases): - #We want to determine for each name if that package was present - #in /all/ the buildroots or just some. - #Because brlist is constructed only from elements of buildroots, we - #can simply check the length + # We want to determine for each name if that package was present + # in /all/ the buildroots or just some. + # Because brlist is constructed only from elements of buildroots, we + # can simply check the length assert len(brlist) <= len(buildroots) if len(brlist) == len(buildroots): - #each buildroot had this as a base package + # each buildroot had this as a base package base.append(name) if len(tags) > 1: log("Warning: found multiple buildroot tags for %s: %s" % (self.nvr, to_list(tags.keys()))) @@ -586,18 +586,18 @@ class BuildTracker(object): self.ignorelist = self.ignorelist + self.excludelist if options.config.has_option('rules', 'substitutions'): - #At present this is a simple multi-line format - #one substitution per line - #format: + # At present this is a simple multi-line format + # one substitution per line + # format: # missing-build build-to-substitute - #TODO: allow more robust substitutions + # TODO: allow more robust substitutions for line in options.config.get('rules', 'substitutions').splitlines(): line = line.strip() if line[:1] == "#": - #skip comment + # skip comment continue if not line: - #blank + # blank continue data = line.split() if len(data) != 2: @@ -633,17 +633,17 @@ class BuildTracker(object): """find out which build is newer""" rc = rpm.labelCompare(nvr1, nvr2) if rc == 1: - #first evr wins + # first evr wins return 1 elif rc == 0: - #same evr + # same evr return 0 else: - #second evr wins + # second evr wins return -1 def newerBuild(self, build, tag): - #XXX: secondary arches need a policy to say if we have newer build localy it will be the substitute + # XXX: secondary arches need a policy to say if we have newer build localy it will be the substitute localBuilds = session.listTagged(tag, inherit=True, package=str(build.name)) newer = None parentevr = (str(build.epoch), build.version, build.release) @@ -657,7 +657,7 @@ class BuildTracker(object): newer = b else: break - #the local is newer + # the local is newer if newer is not None: info = session.getBuild("%s-%s-%s" % (str(newer['name']), newer['version'], newer['release'])) if info: @@ -669,16 +669,16 @@ class BuildTracker(object): def getSubstitute(self, nvr): build = self.substitute_idx.get(nvr) if not build: - #see if remote has it + # see if remote has it info = remote.getBuild(nvr) if info: - #see if we're already tracking it + # see if we're already tracking it build = self.builds.get(info['id']) if not build: build = TrackedBuild(info['id'], tracker=self) else: - #remote doesn't have it - #see if we have it locally + # remote doesn't have it + # see if we have it locally info = session.getBuild(nvr) if info: build = LocalBuild(info) @@ -689,13 +689,13 @@ class BuildTracker(object): def scanBuild(self, build_id, from_build=None, depth=0, tag=None): """Recursively scan a build and its dependencies""" - #print build_id + # print build_id build = self.builds.get(build_id) if build: - #already scanned + # already scanned if from_build: build.addChild(from_build.id) - #There are situations where, we'll need to go forward anyway: + # There are situations where, we'll need to go forward anyway: # - if we were greylisted before, and depth > 0 now # - if we're being substituted and depth is 0 if not (depth > 0 and build.state == 'grey') \ @@ -719,20 +719,20 @@ class BuildTracker(object): return build check = self.checkFilter(build, grey=None) if check is None: - #greylisted builds are ok as deps, but not primary builds + # greylisted builds are ok as deps, but not primary builds if depth == 0: log ("%sGreylisted build %s%s" % (head, build.nvr, tail)) build.setState('grey') return build - #get rid of 'grey' state (filter will not be checked again) + # get rid of 'grey' state (filter will not be checked again) build.updateState() elif not check: log ("%sBlocked build %s%s" % (head, build.nvr, tail)) build.setState('blocked') return build - #make sure we dont have the build name protected + # make sure we dont have the build name protected if build.name not in self.protectlist: - #check to see if a substition applies + # check to see if a substition applies replace = self.substitutions.get(build.nvr) if replace: build.substitute = replace @@ -748,7 +748,7 @@ class BuildTracker(object): else: log ("%sProtected Build: %s" % (head, build.nvr)) if build.state == "common": - #we're good + # we're good if build.rebuilt: log ("%sCommon build (rebuilt) %s%s" % (head, build.nvr, tail)) else: @@ -756,9 +756,9 @@ class BuildTracker(object): elif build.state == 'pending': log ("%sRebuild in progress: %s%s" % (head, build.nvr, tail)) elif build.state == "broken": - #The build already exists locally, but is somehow invalid. - #We should not replace it automatically. An admin can reset it - #if that is the correct thing. A substitution might also be in order + # The build already exists locally, but is somehow invalid. + # We should not replace it automatically. An admin can reset it + # if that is the correct thing. A substitution might also be in order log ("%sWarning: build exists, but is invalid: %s%s" % (head, build.nvr, tail)) # # !! Cases where importing a noarch is /not/ ok must occur @@ -769,13 +769,13 @@ class BuildTracker(object): elif options.import_noarch_only and not build.isNoarch(): log ("%sSkipping archful build: %s" % (head, build.nvr)) elif build.state == "noroot": - #Can't rebuild it, this is what substitutions are for + # Can't rebuild it, this is what substitutions are for log ("%sWarning: no buildroot data for %s%s" % (head, build.nvr, tail)) elif build.state == 'brokendeps': - #should not be possible at this point + # should not be possible at this point log ("Error: build reports brokendeps state before dep scan") elif build.state == "missing": - #scan its deps + # scan its deps log ("%sMissing build %s%s. Scanning deps..." % (head, build.nvr, tail)) newdeps = [] # include extra local builds as deps. @@ -788,7 +788,7 @@ class BuildTracker(object): newdeps.append(extradep) else: log ("%s Warning: could not find build for %s" % (head, dep)) - #don't actually set build.revised_deps until we finish the dep scan + # don't actually set build.revised_deps until we finish the dep scan for dep_id in build.deps: dep = self.scanBuild(dep_id, from_build=build, depth=depth+1, tag=tag) if dep.name in self.ignorelist: @@ -800,14 +800,14 @@ class BuildTracker(object): if isinstance(dep2, TrackedBuild): self.scanBuild(dep2.id, from_build=build, depth=depth+1, tag=tag) elif dep2 is None: - #dep is missing on both local and remote + # dep is missing on both local and remote log ("%sSubstitute dep unavailable: %s" % (head, dep2.nvr)) - #no point in continuing + # no point in continuing break - #otherwise dep2 should be LocalBuild instance + # otherwise dep2 should be LocalBuild instance newdeps.append(dep2) elif dep.state in ('broken', 'brokendeps', 'noroot', 'blocked'): - #no point in continuing + # no point in continuing build.setState('brokendeps') log ("%sCan't rebuild %s, %s is %s" % (head, build.nvr, dep.nvr, dep.state)) newdeps = None @@ -819,7 +819,7 @@ class BuildTracker(object): self.rebuild_order += 1 build.order = self.rebuild_order build.revised_deps = newdeps - #scanning takes a long time, might as well start builds if we can + # scanning takes a long time, might as well start builds if we can self.checkJobs(tag) self.rebuildMissing() if len(self.builds) % 50 == 0: @@ -848,7 +848,7 @@ class BuildTracker(object): """Import an rpm directly from a url""" serverdir = _unique_path('koji-shadow') if options.link_imports: - #bit of a hack, but faster than uploading + # bit of a hack, but faster than uploading dst = "%s/%s/%s" % (koji.pathinfo.work(), serverdir, fn) old_umask = os.umask(0o02) try: @@ -863,8 +863,8 @@ class BuildTracker(object): finally: os.umask(old_umask) else: - #TODO - would be possible, using uploadFile directly, to upload without writing locally. - #for now, though, just use uploadWrapper + # TODO - would be possible, using uploadFile directly, to upload without writing locally. + # for now, though, just use uploadWrapper koji.ensuredir(options.workpath) dst = "%s/%s" % (options.workpath, fn) log ("Downloading %s to %s..." % (url, dst)) @@ -881,7 +881,7 @@ class BuildTracker(object): '''import a build from remote hub''' if not build.srpm: log ("No srpm for build %s, skipping import" % build.nvr) - #TODO - support no-src imports here + # TODO - support no-src imports here return False if not options.remote_topurl: log ("Skipping import of %s, remote_topurl not specified" % build.nvr) @@ -893,7 +893,7 @@ class BuildTracker(object): self._importURL(url, fname) for rpminfo in build.rpms: if rpminfo['arch'] == 'src': - #already imported above + # already imported above continue relpath = pathinfo.rpm(rpminfo) url = "%s/%s" % (build_url, relpath) @@ -906,35 +906,35 @@ class BuildTracker(object): def rebuild(self, build): """Rebuild a remote build using closest possible buildroot""" - #first check that we can + # first check that we can if build.state != 'missing': - log ("Can't rebuild %s. state=%s" % (build.nvr, build.state)) + log("Can't rebuild %s. state=%s" % (build.nvr, build.state)) return - #deps = [] - #for build_id in build.deps: - # dep = self.builds.get(build_id) - # if not dep: - # log ("Missing dependency %i for %s. Not scanned?" % (build_id, build.nvr)) - # return - # if dep.state != 'common': - # log ("Dependency missing for %s: %s (%s)" % (build.nvr, dep.nvr, dep.state)) - # return - # deps.append(dep) + # deps = [] + # for build_id in build.deps: + # dep = self.builds.get(build_id) + # if not dep: + # log ("Missing dependency %i for %s. Not scanned?" % (build_id, build.nvr)) + # return + # if dep.state != 'common': + # log ("Dependency missing for %s: %s (%s)" % (build.nvr, dep.nvr, dep.state)) + # return + # deps.append(dep) deps = build.revised_deps if deps is None: - log ("Can't rebuild %s" % build.nvr) + log("Can't rebuild %s" % build.nvr) return if options.test: - log ("Skipping rebuild of %s (test mode)" % build.nvr) + log("Skipping rebuild of %s (test mode)" % build.nvr) return - #check/create tag + # check/create tag our_tag = "SHADOWBUILD-%s" % build.br_tag taginfo = session.getTag(our_tag) parents = None if not taginfo: - #XXX - not sure what is best here - #how do we pick arches? for now just hardcoded - #XXX this call for perms is stupid, but it's all we've got + # XXX - not sure what is best here + # how do we pick arches? for now just hardcoded + # XXX this call for perms is stupid, but it's all we've got perm_id = None for data in session.getAllPerms(): if data['name'] == 'admin': @@ -942,9 +942,9 @@ class BuildTracker(object): break session.createTag(our_tag, perm=perm_id, arches=options.arches) taginfo = session.getTag(our_tag, strict=True) - #we don't need a target, we trigger our own repo creation and - #pass that repo_id to the build call - #session.createBuildTarget(taginfo['name'], taginfo['id'], taginfo['id']) + # we don't need a target, we trigger our own repo creation and + # pass that repo_id to the build call + # session.createBuildTarget(taginfo['name'], taginfo['id'], taginfo['id']) # duplicate also extra information for a tag (eg. packagemanager setting) rtaginfo = remote.getTag(build.br_tag) if 'extra' in rtaginfo: @@ -955,7 +955,7 @@ class BuildTracker(object): parents = session.getInheritanceData(taginfo['id']) if parents: log ("Warning: shadow build tag has inheritance") - #check package list + # check package list pkgs = {} for pkg in session.listPackages(tagID=taginfo['id']): pkgs[pkg['package_name']] = pkg @@ -963,7 +963,7 @@ class BuildTracker(object): for dep in deps: name = dep.info['name'] if name not in pkgs: - #guess owner + # guess owner owners = {} for pkg in session.listPackages(pkgID=name): owners.setdefault(pkg['owner_id'], []).append(pkg) @@ -972,36 +972,36 @@ class BuildTracker(object): order.sort() owner = order[-1][1] else: - #just use ourselves + # just use ourselves owner = session.getLoggedInUser()['id'] missing_pkgs.append((name, owner)) - #check build list + # check build list cur_builds = {} for binfo in session.listTagged(taginfo['id']): - #index by name in tagging order (latest first) + # index by name in tagging order (latest first) cur_builds.setdefault(binfo['name'], []).append(binfo) to_untag = [] to_tag = [] for dep in deps: - #XXX - assuming here that there is only one dep per 'name' + # XXX - assuming here that there is only one dep per 'name' # may want to check that this is true cur_order = cur_builds.get(dep.info['name'], []) tagged = False for binfo in cur_order: if binfo['nvr'] == dep.nvr: tagged = True - #may not be latest now, but it will be after we do all the untagging + # may not be latest now, but it will be after we do all the untagging else: # note that the untagging keeps older builds from piling up. In a sense # we're gc-pruning this tag ourselves every pass. to_untag.append(binfo) if not tagged: to_tag.append(dep) - #TODO - "add-on" packages + # TODO - "add-on" packages # for handling arch-specific deps that may not show up on remote # e.g. elilo or similar # these extra packages should be added to tag, but not the build group - #TODO - local extra builds + # TODO - local extra builds # a configurable mechanism to add specific local builds to the buildroot drop_groups = [] build_group = None @@ -1013,12 +1013,12 @@ class BuildTracker(object): log ("Warning: found stray group: %s" % group) drop_groups.append(group['name']) if build_group: - #fix build group package list based on base of build to shadow + # fix build group package list based on base of build to shadow needed = dict([(n, 1) for n in build.base]) current = dict([(p['package'], 1) for p in build_group['packagelist']]) add_pkgs = [n for n in needed if n not in current] drop_pkgs = [n for n in current if n not in needed] - #no group deps needed/allowed + # no group deps needed/allowed drop_deps = [(g['name'], 1) for g in build_group['grouplist']] if drop_deps: log ("Warning: build group had deps: %r" % build_group) @@ -1026,8 +1026,8 @@ class BuildTracker(object): add_pkgs = build.base drop_pkgs = [] drop_deps = [] - #update package list, tagged packages, and groups in one multicall/transaction - #(avoid useless repo regens) + # update package list, tagged packages, and groups in one multicall/transaction + # (avoid useless repo regens) session.multicall = True for name, owner in missing_pkgs: session.packageListAdd(taginfo['id'], name, owner=owner) @@ -1035,35 +1035,35 @@ class BuildTracker(object): session.untagBuildBypass(taginfo['id'], binfo['id']) for dep in to_tag: session.tagBuildBypass(taginfo['id'], dep.nvr) - #shouldn't need force here - #set groups data + # shouldn't need force here + # set groups data if not build_group: # build group not present. add it session.groupListAdd(taginfo['id'], 'build', force=True) - #using force in case group is blocked. This shouldn't be the case, but... + # using force in case group is blocked. This shouldn't be the case, but... for pkg_name in drop_pkgs: - #in principal, our tag should not have inheritance, so the remove call is the right thing + # in principal, our tag should not have inheritance, so the remove call is the right thing session.groupPackageListRemove(taginfo['id'], 'build', pkg_name) for pkg_name in add_pkgs: session.groupPackageListAdd(taginfo['id'], 'build', pkg_name) - #we never add any blocks, so forcing shouldn't be required - #TODO - adjust extra_arches for package to build - #get event id to facilitate waiting on repo + # we never add any blocks, so forcing shouldn't be required + # TODO - adjust extra_arches for package to build + # get event id to facilitate waiting on repo # not sure if getLastEvent is good enough # short of adding a new call, perhaps use getLastEvent together with event of # current latest repo for tag session.getLastEvent() results = session.multiCall(strict=True) event_id = results[-1][0]['id'] - #TODO - verify / check results ? + # TODO - verify / check results ? task_id = session.newRepo(our_tag, event=event_id) - #TODO - upload src + # TODO - upload src # [?] use remote SCM url (if avail)? src = build.getSource() if not src: log ("Couldn't get source for %s" % build.nvr) return None - #wait for repo task + # wait for repo task log ("Waiting on newRepo task %i" % task_id) while True: tinfo = session.getTaskInfo(task_id) @@ -1073,10 +1073,10 @@ class BuildTracker(object): elif tstate in ('CANCELED', 'FAILED'): log ("Error: failed to generate repo") return None - #add a timeout? - #TODO ...and verify repo + # add a timeout? + # TODO ...and verify repo repo_id, event_id = session.getTaskResult(task_id) - #kick off build + # kick off build task_id = session.build(src, None, opts={'repo_id': repo_id}, priority=options.priority) return task_id @@ -1090,34 +1090,34 @@ class BuildTracker(object): log("%s: %i (+%i replaced)" % (state, len(not_replaced), n_replaced)) if not_replaced and len(not_replaced) < 8: log(' '.join([b.nvr for b in not_replaced])) - #generate a report of the most frequent problem deps + # generate a report of the most frequent problem deps problem_counts = {} for build in self.state_idx['brokendeps'].values(): for dep_id in build.deps: dep = self.builds.get(dep_id) if not dep: - #unscanned - #possible because we short circuit the earlier scan on problems - #we don't really know if this one is a problem or not, so just - #skip it. + # unscanned + # possible because we short circuit the earlier scan on problems + # we don't really know if this one is a problem or not, so just + # skip it. continue if dep.state in ('common', 'pending', 'missing'): - #not a problem + # not a problem continue nvr = dep.nvr if dep.substitute: dep2 = self.getSubstitute(dep.substitute) if dep2: - #we have a substitution, so not a problem + # we have a substitution, so not a problem continue - #otherwise the substitution is the problem + # otherwise the substitution is the problem nvr = dep.substitute problem_counts.setdefault(nvr, 0) problem_counts[nvr] += 1 order = [(c, nvr) for (nvr, c) in six.iteritems(problem_counts)] if order: order.sort(reverse=True) - #print top 5 problems + # print top 5 problems log("-- top problems --") for (c, nvr) in order[:5]: log(" %s (%i)" % (nvr, c)) @@ -1138,7 +1138,7 @@ class BuildTracker(object): """Check outstanding jobs. Return true if anything changes""" ret = False for build_id, build in self.state_idx['pending'].items(): - #check pending builds + # check pending builds if not build.task_id: log ("No task id recorded for %s" % build.nvr) build.updateState() @@ -1152,8 +1152,8 @@ class BuildTracker(object): state = koji.TASK_STATES[info['state']] if state in ('CANCELED', 'FAILED'): log ("Task %i is %s (build %s)" % (build.task_id, state, build.nvr)) - #we have to set the state to broken manually (updateState will mark - #a failed build as missing) + # we have to set the state to broken manually (updateState will mark + # a failed build as missing) build.setState('broken') ret = True elif state == 'CLOSED': @@ -1168,9 +1168,9 @@ class BuildTracker(object): return ret def checkBuildDeps(self, build): - #check deps + # check deps if build.revised_deps is None: - #log ("No revised deplist yet for %s" % build.nvr) + # log("No revised deplist yet for %s" % build.nvr) return False problem = [x for x in build.revised_deps if x.state in ('broken', 'brokendeps', 'noroot', 'blocked')] @@ -1182,10 +1182,10 @@ class BuildTracker(object): not_common = [x for x in build.revised_deps if x.state not in ('common', 'local')] if not_common: - #could be missing or still building or whatever - #log ("Still missing %i revised deps for %s" % (len(not_common), build.nvr)) + # could be missing or still building or whatever + # log("Still missing %i revised deps for %s" % (len(not_common), build.nvr)) return False - #otherwise, we should be good to rebuild + # otherwise, we should be good to rebuild return True def rebuildMissing(self): @@ -1200,15 +1200,15 @@ class BuildTracker(object): for order, build_id, build in missing: if not self.checkBuildDeps(build): continue - #otherwise, we should be good to rebuild + # otherwise, we should be good to rebuild log ("rebuild: %s" % build.nvr) task_id = self.rebuild(build) ret = True if options.test: - #pretend build is available + # pretend build is available build.setState('common') elif not task_id: - #something went wrong setting up the rebuild + # something went wrong setting up the rebuild log ("Did not get a task for %s" % build.nvr) build.setState('broken') else: @@ -1225,14 +1225,14 @@ class BuildTracker(object): def runRebuilds(self, tag=None): """Rebuild missing builds""" log ("Determining rebuild order") - #using self.state_idx to track build states - #make sure state_idx has at least these states + # using self.state_idx to track build states + # make sure state_idx has at least these states initial_avail = len(self.state_idx['common']) self.report_brief() while True: if (not self.state_idx['missing'] and not self.state_idx['pending']) or \ (options.prefer_new and not self.state_idx['pending']): - #we're done + # we're done break changed1 = self.checkJobs(tag) changed2 = self.rebuildMissing() @@ -1244,7 +1244,7 @@ class BuildTracker(object): def tagSuccessful(self, nvr, tag): """tag completed builds into final tags""" - #TODO: check if there are other reasons why tagging may fail and handle them + # TODO: check if there are other reasons why tagging may fail and handle them try: session.tagBuildBypass(tag, nvr) log ("tagged %s to %s" % (nvr, tag)) @@ -1294,8 +1294,8 @@ if __name__ == "__main__": session = koji.ClientSession(options.server, session_opts) if not options.noauth: activate_session(session) - #XXX - sane auth - #XXX - config! + # XXX - sane auth + # XXX - config! remote_opts = {'anon_retry': True} for k in ('debug_xmlrpc', 'debug'): remote_opts[k] = getattr(options, k) @@ -1309,7 +1309,7 @@ if __name__ == "__main__": pass except SystemExit: rv = 1 - #except: + # except: # if options.debug: # raise # else: diff --git a/util/koji-sweep-db b/util/koji-sweep-db index decf4e1..b1896bc 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -169,7 +169,7 @@ if __name__ == "__main__": config.read(options.conf) cfgmap = [ - #option, type, default + # option, type, default ['DBName', 'string', None], ['DBUser', 'string', None], ['DBHost', 'string', None], From ce1f9928afdab5f0124ad93cbfb98f0e63efdeaa Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:21 +0000 Subject: [PATCH 7/23] flake8: apply E1 rules --- diff --git a/.flake8 b/.flake8 index 27b4d7f..b20a51c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -select = I,C,F,E265 +select = I,C,F,E1,E265 exclude = .git, __pycache__, diff --git a/builder/kojid b/builder/kojid index a775b5d..fd607a7 100755 --- a/builder/kojid +++ b/builder/kojid @@ -231,7 +231,7 @@ class BuildRoot(object): self.tag_name = self.config['name'] if self.config['id'] != repo_info['tag_id']: raise koji.BuildrootError("tag/repo mismatch: %s vs %s" \ - % (self.config['name'], repo_info['tag_name'])) + % (self.config['name'], repo_info['tag_name'])) repo_state = koji.REPO_STATES[repo_info['state']] if repo_state == 'EXPIRED': # This should be ok. Expired repos are still intact, just not @@ -698,7 +698,7 @@ class BuildRoot(object): path_comps = relpath.split('/') if len(path_comps) < 3: raise koji.BuildrootError('files found in unexpected path in local Maven repo, directory: %s, files: %s' % \ - (relpath, ', '.join([f['filename'] for f in maven_files]))) + (relpath, ', '.join([f['filename'] for f in maven_files]))) # extract the Maven info from the path within the local repo maven_info = {'version': path_comps[-1], 'artifact_id': path_comps[-2], @@ -966,7 +966,7 @@ class BuildTask(BaseTaskHandler): # make sure specified repo matches target if repo_info['tag_id'] != target_info['build_tag']: raise koji.BuildError('Repo/Target mismatch: %s/%s' \ - % (repo_info['tag_name'], target_info['build_tag_name'])) + % (repo_info['tag_name'], target_info['build_tag_name'])) else: # if repo_id is specified, we can allow the 'target' arg to simply specify # the destination tag (since the repo specifies the build tag). @@ -1022,10 +1022,10 @@ class BuildTask(BaseTaskHandler): # Make sure package is on the list for this tag if pkg_cfg is None: raise koji.BuildError("package %s not in list for tag %s" \ - % (data['name'], target_info['dest_tag_name'])) + % (data['name'], target_info['dest_tag_name'])) elif pkg_cfg['blocked']: raise koji.BuildError("package %s is blocked for tag %s" \ - % (data['name'], target_info['dest_tag_name'])) + % (data['name'], target_info['dest_tag_name'])) # TODO - more pre tests archlist = self.getArchList(build_tag, h, extra=extra_arches) # let the system know about the build we're attempting @@ -1034,11 +1034,11 @@ class BuildTask(BaseTaskHandler): build_id = self.session.host.initBuild(data) # (initBuild raises an exception if there is a conflict) failany = (self.opts.get('fail_fast', False) - or not getattr(self.options, 'build_arch_can_fail', False)) + or not getattr(self.options, 'build_arch_can_fail', False)) try: self.extra_information = { "src": src, "data": data, "target": target } srpm,rpms,brmap,logs = self.runBuilds(srpm, build_tag, archlist, - repo_info['id'], failany=failany) + repo_info['id'], failany=failany) if opts.get('scratch'): # scratch builds do not get imported @@ -1192,8 +1192,8 @@ class BuildTask(BaseTaskHandler): archlist = [ a for a in archlist if a not in excludearch ] if not archlist: raise koji.BuildError("No valid arches were found. tag %r, " - "exclusive %r, exclude %r" % (tag_arches, - exclusivearch, excludearch)) + "exclusive %r, exclude %r" % (tag_arches, + exclusivearch, excludearch)) if set(archlist) != set(tag_arches): return random.choice(archlist) else: @@ -1256,10 +1256,10 @@ class BuildTask(BaseTaskHandler): # create the tagBuild subtask # this will handle the "post tests" task_id = self.session.host.subtask(method='tagBuild', - arglist=[dest_tag,build_id,False,None,True], - label='tag', - parent=self.id, - arch='noarch') + arglist=[dest_tag,build_id,False,None,True], + label='tag', + parent=self.id, + arch='noarch') self.wait(task_id) @@ -1506,10 +1506,10 @@ class MavenTask(MultiPlatformTask): # Make sure package is on the list for this tag if dest_cfg is None: raise koji.BuildError("package %s not in list for tag %s" \ - % (build_info['name'], dest_tag['name'])) + % (build_info['name'], dest_tag['name'])) elif dest_cfg['blocked']: raise koji.BuildError("package %s is blocked for tag %s" \ - % (build_info['name'], dest_tag['name'])) + % (build_info['name'], dest_tag['name'])) build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info) self.build_id = build_info['id'] @@ -1666,8 +1666,8 @@ class BuildMavenTask(BaseBuildTask): if self.opts.get('patches'): # filter out directories and files beginning with . (probably scm metadata) patches = [patch for patch in os.listdir(patchcheckoutdir) if \ - os.path.isfile(os.path.join(patchcheckoutdir, patch)) and \ - patch.endswith('.patch')] + os.path.isfile(os.path.join(patchcheckoutdir, patch)) and \ + patch.endswith('.patch')] if not patches: raise koji.BuildError('no patches found at %s' % self.opts.get('patches')) patches.sort() @@ -2018,10 +2018,10 @@ class WrapperRPMTask(BaseBuildTask): # Make sure package is on the list for this tag if pkg_cfg is None: raise koji.BuildError("package %s not in list for tag %s" \ - % (data['name'], build_target['dest_tag_name'])) + % (data['name'], build_target['dest_tag_name'])) elif pkg_cfg['blocked']: raise koji.BuildError("package %s is blocked for tag %s" \ - % (data['name'], build_target['dest_tag_name'])) + % (data['name'], build_target['dest_tag_name'])) self.new_build_id = self.session.host.initBuild(data) try: @@ -2046,7 +2046,7 @@ class WrapperRPMTask(BaseBuildTask): if self.new_build_id: self.session.host.failBuild(self.id, self.new_build_id) raise koji.BuildError('multiple srpms found in %s: %s, %s' % \ - (resultdir, srpm, filename)) + (resultdir, srpm, filename)) elif filename.endswith('.rpm'): rpms.append(filename) elif filename.endswith('.log'): @@ -2055,7 +2055,7 @@ class WrapperRPMTask(BaseBuildTask): if self.new_build_id: self.session.host.failBuild(self.id, self.new_build_id) raise koji.BuildError('unexpected file found in %s: %s' % \ - (resultdir, filename)) + (resultdir, filename)) if not srpm: if self.new_build_id: @@ -2139,10 +2139,10 @@ class ChainMavenTask(MultiPlatformTask): # Make sure package is on the list for this tag if dest_cfg is None: raise koji.BuildError("package %s not in list for tag %s" \ - % (package, dest_tag['name'])) + % (package, dest_tag['name'])) elif dest_cfg['blocked']: raise koji.BuildError("package %s is blocked for tag %s" \ - % (package, dest_tag['name'])) + % (package, dest_tag['name'])) self.depmap = {} for package, params in builds.items(): @@ -2357,7 +2357,7 @@ class BuildImageTask(MultiPlatformTask): def initImageBuild(self, name, version, release, target_info, opts): """create a build object for this image build""" pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'], - name) + name) self.logger.debug("%r" % pkg_cfg) if not opts.get('skip_tag') and not opts.get('scratch'): # Make sure package is on the list for this tag @@ -2366,7 +2366,7 @@ class BuildImageTask(MultiPlatformTask): elif pkg_cfg['blocked']: raise koji.BuildError("package (image) %s is blocked for tag %s" % (name, target_info['dest_tag_name'])) return self.session.host.initImageBuild(self.id, - dict(name=name, version=version, release=release, epoch=0)) + dict(name=name, version=version, release=release, epoch=0)) def getRelease(self, name, ver): """return the next available release number for an N-V""" @@ -2408,7 +2408,7 @@ class BuildBaseImageTask(BuildImageTask): raise koji.ApplianceError('The Release may not have a hyphen') if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, - target_info, opts) + target_info, opts) subtasks = {} self.logger.debug("Spawning jobs for image arches: %r" % (arches)) @@ -2418,7 +2418,7 @@ class BuildBaseImageTask(BuildImageTask): subtasks[arch] = self.session.host.subtask( method='createImage', arglist=[name, version, release, arch, target_info, - build_tag, repo_info, inst_url, opts], + build_tag, repo_info, inst_url, opts], label=arch, parent=self.id, arch=arch) if arch in opts.get('optional_arches', []): canfail.append(subtasks[arch]) @@ -2476,7 +2476,7 @@ class BuildBaseImageTask(BuildImageTask): self.session.host.moveImageBuildToScratch(self.id, results) else: self.session.host.completeImageBuild(self.id, bld_info['id'], - results) + results) except (SystemExit,ServerExit,KeyboardInterrupt): # we do not trap these @@ -2492,8 +2492,8 @@ class BuildBaseImageTask(BuildImageTask): # tag it if not opts.get('scratch') and not opts.get('skip_tag'): tag_task_id = self.session.host.subtask(method='tagBuild', - arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], - label='tag', parent=self.id, arch='noarch') + arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], + label='tag', parent=self.id, arch='noarch') self.wait(tag_task_id) # report results @@ -2540,11 +2540,11 @@ class BuildApplianceTask(BuildImageTask): release = self.getRelease(name, version) if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, - target_info, opts) + target_info, opts) create_task_id = self.session.host.subtask(method='createAppliance', - arglist=[name, version, release, arch, target_info, build_tag, - repo_info, ksfile, opts], - label='appliance', parent=self.id, arch=arch) + arglist=[name, version, release, arch, target_info, build_tag, + repo_info, ksfile, opts], + label='appliance', parent=self.id, arch=arch) results = self.wait(create_task_id) self.logger.info('image build task (%s) completed' % create_task_id) self.logger.info('results: %s' % results) @@ -2578,14 +2578,14 @@ class BuildApplianceTask(BuildImageTask): # tag it if not opts.get('scratch') and not opts.get('skip_tag'): tag_task_id = self.session.host.subtask(method='tagBuild', - arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], - label='tag', parent=self.id, arch='noarch') + arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], + label='tag', parent=self.id, arch='noarch') self.wait(tag_task_id) # report results if opts.get('scratch'): respath = os.path.join(koji.pathinfo.work(), - koji.pathinfo.taskrelpath(create_task_id)) + koji.pathinfo.taskrelpath(create_task_id)) report = 'Scratch ' else: respath = koji.pathinfo.imagebuild(bld_info) @@ -2613,7 +2613,7 @@ class BuildLiveCDTask(BuildImageTask): opts = {} if not image_enabled: self.logger.error("LiveCD features require the following dependencies: " - "pykickstart, pycdio, and possibly python-hashlib") + "pykickstart, pycdio, and possibly python-hashlib") raise koji.LiveCDError('LiveCD functions not available') # build the image @@ -2624,11 +2624,11 @@ class BuildLiveCDTask(BuildImageTask): release = self.getRelease(name, version) if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, - target_info, opts) + target_info, opts) create_task_id = self.session.host.subtask(method='createLiveCD', - arglist=[name, version, release, arch, target_info, build_tag, - repo_info, ksfile, opts], - label='livecd', parent=self.id, arch=arch) + arglist=[name, version, release, arch, target_info, build_tag, + repo_info, ksfile, opts], + label='livecd', parent=self.id, arch=arch) results = self.wait(create_task_id) self.logger.info('image build task (%s) completed' % create_task_id) self.logger.info('results: %s' % results) @@ -2662,14 +2662,14 @@ class BuildLiveCDTask(BuildImageTask): # tag it if necessary if not opts.get('scratch') and not opts.get('skip_tag'): tag_task_id = self.session.host.subtask(method='tagBuild', - arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], - label='tag', parent=self.id, arch='noarch') + arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], + label='tag', parent=self.id, arch='noarch') self.wait(tag_task_id) # report the results if opts.get('scratch'): respath = os.path.join(koji.pathinfo.work(), - koji.pathinfo.taskrelpath(create_task_id)) + koji.pathinfo.taskrelpath(create_task_id)) report = 'Scratch ' else: respath = koji.pathinfo.imagebuild(bld_info) @@ -2704,7 +2704,7 @@ class BuildLiveMediaTask(BuildImageTask): if not image_enabled: # XXX - are these still required here? self.logger.error("Missing the following dependencies: " - "pykickstart, pycdio, and possibly python-hashlib") + "pykickstart, pycdio, and possibly python-hashlib") raise koji.PreBuildError('Live Media functions not available') # build the image @@ -2715,14 +2715,14 @@ class BuildLiveMediaTask(BuildImageTask): release = self.getRelease(name, version) if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, - target_info, opts) + target_info, opts) subtasks = {} canfail = [] for arch in arches: subtasks[arch] = self.subtask('createLiveMedia', - [name, version, release, arch, target_info, build_tag, - repo_info, ksfile, opts], - label='livemedia %s' % arch, arch=arch) + [name, version, release, arch, target_info, build_tag, + repo_info, ksfile, opts], + label='livemedia %s' % arch, arch=arch) if arch in opts.get('optional_arches', []): canfail.append(subtasks[arch]) self.logger.debug("Tasks that can fail: %r", canfail) @@ -2762,9 +2762,9 @@ class BuildLiveMediaTask(BuildImageTask): if arch in ignored_arches: continue arglist = [spec_url, target_info, bld_info, tinfo, - {'repo_id': repo_info['id']}] + {'repo_id': repo_info['id']}] wrapper_tasks[arch] = self.subtask('wrapperRPM', arglist, - label='wrapper %s' % arch, arch='noarch') + label='wrapper %s' % arch, arch='noarch') results2 = self.wait(to_list(wrapper_tasks.values()), all=True, failany=True) self.logger.debug('wrapper results: %r', results2) @@ -2800,8 +2800,8 @@ class BuildLiveMediaTask(BuildImageTask): # tag it if necessary if not opts.get('scratch') and not opts.get('skip_tag'): tag_task_id = self.session.host.subtask(method='tagBuild', - arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], - label='tag', parent=self.id, arch='noarch') + arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], + label='tag', parent=self.id, arch='noarch') self.wait(tag_task_id) # report the results @@ -2919,7 +2919,7 @@ class ImageTask(BaseTaskHandler): "'%s' : %s" % (kspath, e)) except kserrors.KickstartError as e: raise koji.LiveCDError("Failed to parse kickstart file " - "'%s' : %s" % (kspath, e)) + "'%s' : %s" % (kspath, e)) def prepareKickstart(self, repo_info, target_info, arch, broot, opts): """ @@ -3002,27 +3002,27 @@ class ImageTask(BaseTaskHandler): # Duplicated with pungi-fedora fedora.conf # see https://pagure.io/koji/pull-request/817 substitutions = { - 'Beta': 'B', - 'Rawhide': 'rawh', - 'Astronomy_KDE': 'AstK', - 'Atomic': 'AH', - 'Cinnamon': 'Cinn', - 'Cloud': 'C', - 'Design_suite': 'Dsgn', - 'Electronic_Lab': 'Elec', - 'Everything': 'E', + 'Beta': 'B', + 'Rawhide': 'rawh', + 'Astronomy_KDE': 'AstK', + 'Atomic': 'AH', + 'Cinnamon': 'Cinn', + 'Cloud': 'C', + 'Design_suite': 'Dsgn', + 'Electronic_Lab': 'Elec', + 'Everything': 'E', 'Games': 'Game', 'Images': 'img', 'Jam_KDE': 'Jam', - 'MATE_Compiz': 'MATE', - # Note https://pagure.io/pungi-fedora/issue/533 - 'Python-Classroom': 'Clss', - 'Python_Classroom': 'Clss', + 'MATE_Compiz': 'MATE', + # Note https://pagure.io/pungi-fedora/issue/533 + 'Python-Classroom': 'Clss', + 'Python_Classroom': 'Clss', 'Robotics': 'Robo', - 'Scientific_KDE': 'SciK', + 'Scientific_KDE': 'SciK', 'Security': 'Sec', 'Server': 'S', - 'Workstation': 'WS', + 'Workstation': 'WS', 'WorkstationOstree': 'WS', } @@ -3125,7 +3125,7 @@ class ApplianceTask(ImageTask): if not opts.get('scratch'): hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), - cachedir[1:])) + cachedir[1:])) broot.markExternalRPMs(hdrlist) imgdata['rpmlist'] = hdrlist @@ -3216,7 +3216,7 @@ class LiveCDTask(ImageTask): self.opts = opts broot = self.makeImgBuildRoot(build_tag, repo_info, arch, - 'livecd-build') + 'livecd-build') kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) @@ -3268,19 +3268,19 @@ class LiveCDTask(ImageTask): self.uploadFile(isosrc, remoteName=isoname) imgdata = {'arch': arch, - 'files': [isoname], - 'rootdev': None, - 'task_id': self.id, - 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'livecd.log', os.path.basename(ksfile), - os.path.basename(kskoji)], - 'name': name, - 'version': version, - 'release': release - } + 'files': [isoname], + 'rootdev': None, + 'task_id': self.id, + 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', + 'livecd.log', os.path.basename(ksfile), + os.path.basename(kskoji)], + 'name': name, + 'version': version, + 'release': release + } if not opts.get('scratch'): hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), - cachedir[1:])) + cachedir[1:])) imgdata ['rpmlist'] = hdrlist broot.markExternalRPMs(hdrlist) @@ -3397,7 +3397,7 @@ class LiveMediaTask(ImageTask): self.opts = opts broot = self.makeImgBuildRoot(build_tag, repo_info, arch, - 'livemedia-build') + 'livemedia-build') kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) @@ -3415,7 +3415,7 @@ class LiveMediaTask(ImageTask): '--resultdir', resultdir, '--project', name, # '--tmp', '/tmp' - ] + ] volid = opts.get('volid') @@ -3430,12 +3430,12 @@ class LiveMediaTask(ImageTask): cmd.extend(['--make-iso', '--volid', volid, '--iso-only', - ]) + ]) isoname='%s-%s-%s-%s.iso' % (name, arch, version, release) cmd.extend(['--iso-name', isoname, '--releasever', version, - ]) + ]) if arch == 'x86_64': @@ -3453,7 +3453,7 @@ class LiveMediaTask(ImageTask): logdirs = [ os.path.join(broot.tmpdir(), 'lmc-logs'), os.path.join(broot.tmpdir(), 'lmc-logs/anaconda'), - ] + ] for logdir in logdirs: if not os.path.isdir(logdir): continue @@ -3499,16 +3499,16 @@ class LiveMediaTask(ImageTask): self.uploadFile(isosrc, remoteName=isoname) imgdata = {'arch': arch, - 'files': [isoname], - 'rootdev': None, - 'task_id': self.id, - 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'livemedia-out.log', os.path.basename(ksfile), - os.path.basename(kskoji)], - 'name': name, - 'version': version, - 'release': release - } + 'files': [isoname], + 'rootdev': None, + 'task_id': self.id, + 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', + 'livemedia-out.log', os.path.basename(ksfile), + os.path.basename(kskoji)], + 'name': name, + 'version': version, + 'release': release + } if not opts.get('scratch'): # TODO - generate list of rpms in image # (getImagePackages doesn't work here) @@ -3549,7 +3549,7 @@ class OzImageTask(BaseTaskHandler): logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=self.opts.get('scratch')) scmsrcdir = scm.checkout(self.workdir, self.session, - self.getUploadDir(), logfile) + self.getUploadDir(), logfile) self.run_callbacks("postSCMCheckout", scminfo=scm.get_info(), build_tag=build_tag, @@ -3589,10 +3589,10 @@ class OzImageTask(BaseTaskHandler): ks.readKickstart(kspath) except IOError as e: raise koji.BuildError("Failed to read kickstart file " - "'%s' : %s" % (kspath, e)) + "'%s' : %s" % (kspath, e)) except kserrors.KickstartError as e: raise koji.BuildError("Failed to parse kickstart file " - "'%s' : %s" % (kspath, e)) + "'%s' : %s" % (kspath, e)) return ks def prepareKickstart(self, kspath, install_tree): @@ -3627,7 +3627,7 @@ class OzImageTask(BaseTaskHandler): # --repo was not given, so we use the target's build repo path_info = koji.PathInfo(topdir=self.options.topurl) repopath = path_info.repo(self.repo_info['id'], - self.target_info['build_tag_name']) + self.target_info['build_tag_name']) baseurl = '%s/%s' % (repopath, self.arch) self.logger.debug('BASEURL: %s' % baseurl) ks.handler.repo.repoList.append(repo_class( @@ -3686,7 +3686,7 @@ class OzImageTask(BaseTaskHandler): 'rhevm_image_format': 'qcow2', 'tdl_require_root_pw': False, 'image_manager_args': { - 'storage_path': os.path.join(self.workdir, 'output_image')}, + 'storage_path': os.path.join(self.workdir, 'output_image')}, } def makeTemplate(self, name, inst_tree): @@ -3705,9 +3705,9 @@ class OzImageTask(BaseTaskHandler): # TODO: intelligently guess the distro based on the install tree URL distname, distver = self.parseDistro(self.opts.get('distro')) if self.arch in ['armhfp','armv7hnl','armv7hl']: - arch = 'armv7l' + arch = 'armv7l' else: - arch = self.arch + arch = self.arch template = """ -""" % (name, self.opts.get('disk_size')) +""" % (name, self.opts.get('disk_size')) # noqa: E501 return template def parseDistro(self, distro): @@ -3831,7 +3910,9 @@ class BaseImageTask(OzImageTask): Some image formats require others to be processed first, which is why we have to do this. raw files in particular may not be kept. """ - supported = ('raw', 'raw-xz', 'liveimg-squashfs', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', 'vagrant-vmware-fusion', 'vagrant-hyperv', 'vpc', "tar-gz") + supported = ('raw', 'raw-xz', 'liveimg-squashfs', 'vmdk', 'qcow', 'qcow2', 'vdi', + 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', + 'vagrant-vmware-fusion', 'vagrant-hyperv', 'vpc', "tar-gz") for f in formats: if f not in supported: raise koji.ApplianceError('Invalid format: %s' % f) @@ -3945,7 +4026,8 @@ class BaseImageTask(OzImageTask): self.tlog.removeHandler(self.fhandler) self.uploadFile(self.ozlog) if 'No disk activity' in details: - details = 'Automated install failed or prompted for input. See the screenshot in the task results for more information.' + details = 'Automated install failed or prompted for input. ' \ + 'See the screenshot in the task results for more information' raise koji.ApplianceError('Image status is %s: %s' % (status, details)) @@ -4108,8 +4190,8 @@ class BaseImageTask(OzImageTask): if format == 'vagrant-vmware-fusion': format = 'vsphere-ova' img_opts['vsphere_ova_format'] = 'vagrant-vmware-fusion' - # The initial disk image transform for VMWare Fusion/Workstation requires a "standard" VMDK - # not the stream oriented format used for VirtualBox or regular VMWare OVAs + # The initial disk image transform for VMWare Fusion/Workstation requires a "standard" + # VMDK, not the stream oriented format used for VirtualBox or regular VMWare OVAs img_opts['vsphere_vmdk_format'] = 'standard' fixed_params = ['vsphere_ova_format', 'vsphere_vmdk_format'] if format == 'vagrant-hyperv': @@ -4117,7 +4199,8 @@ class BaseImageTask(OzImageTask): img_opts['hyperv_ova_format'] = 'hyperv-vagrant' fixed_params = ['hyperv_ova_format'] targ = self._do_target_image(self.base_img.base_image.identifier, - format.replace('-ova', ''), img_opts=img_opts, fixed_params=fixed_params) + format.replace('-ova', ''), img_opts=img_opts, + fixed_params=fixed_params) targ2 = self._do_target_image(targ.target_image.identifier, 'OVA', img_opts=img_opts, fixed_params=fixed_params) return {'image': targ2.target_image.data} @@ -4166,7 +4249,9 @@ class BaseImageTask(OzImageTask): self._mergeFactoryParams(img_opts, fixed_params) self.logger.debug('img_opts_post_merge: %s' % img_opts) target = self.bd.builder_for_target_image(image_type, - image_id=base_id, template=None, parameters=img_opts) + image_id=base_id, + template=None, + parameters=img_opts) target.target_thread.join() self._checkImageState(target) return target @@ -4206,9 +4291,12 @@ class BaseImageTask(OzImageTask): self.getUploadDir(), logerror=1) return {'image': newimg} - def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None): + def handler(self, name, version, release, arch, target_info, + build_tag, repo_info, inst_tree, opts=None): if not ozif_enabled: - self.logger.error("ImageFactory features require the following dependencies: pykickstart, imagefactory, oz and possibly python-hashlib") + self.logger.error( + "ImageFactory features require the following dependencies: " + "pykickstart, imagefactory, oz and possibly python-hashlib") raise koji.ApplianceError('ImageFactory functions not available') if opts is None: @@ -4339,11 +4427,14 @@ class BuildIndirectionImageTask(OzImageTask): if not opts.get('skip_tag') and not opts.get('scratch'): # Make sure package is on the list for this tag if pkg_cfg is None: - raise koji.BuildError("package (image) %s not in list for tag %s" % (name, target_info['dest_tag_name'])) + raise koji.BuildError("package (image) %s not in list for tag %s" % + (name, target_info['dest_tag_name'])) elif pkg_cfg['blocked']: - raise koji.BuildError("package (image) %s is blocked for tag %s" % (name, target_info['dest_tag_name'])) + raise koji.BuildError("package (image) %s is blocked for tag %s" % + (name, target_info['dest_tag_name'])) return self.session.host.initImageBuild(self.id, - dict(name=name, version=version, release=release, epoch=0)) + dict(name=name, version=version, release=release, + epoch=0)) def getRelease(self, name, ver): """return the next available release number for an N-V""" @@ -4371,7 +4462,8 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=self.opts.get('scratch')) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), + build_tag=build_tag, scratch=self.opts.get('scratch')) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) @@ -4402,11 +4494,13 @@ class BuildIndirectionImageTask(OzImageTask): taskinfo = self.session.getTaskInfo(task_id) taskstate = koji.TASK_STATES[taskinfo['state']].lower() if taskstate != 'closed': - raise koji.BuildError("Input task (%d) must be in closed state - current state is (%s)" % + raise koji.BuildError("Input task (%d) must be in closed state" + " - current state is (%s)" % (task_id, taskstate)) taskmethod = taskinfo['method'] if taskmethod != "createImage": - raise koji.BuildError("Input task method must be 'createImage' - actual method (%s)" % + raise koji.BuildError("Input task method must be 'createImage'" + " - actual method (%s)" % (taskmethod)) result = self.session.getTaskResult(task_id) @@ -4424,7 +4518,9 @@ class BuildIndirectionImageTask(OzImageTask): tdl_full = os.path.join(task_dir, task_tdl) if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)): - raise koji.BuildError("Missing TDL or qcow2 image for task (%d) - possible expired scratch build" % (task_id)) + raise koji.BuildError( + "Missing TDL or qcow2 image for task (%d) - possible expired scratch build" % + (task_id)) # The sequence to recreate a valid persistent image is as follows # Create a new BaseImage object @@ -4445,7 +4541,10 @@ class BuildIndirectionImageTask(OzImageTask): return factory_base_image def _nvr_to_image(nvr, arch): - """ Take a build ID or NVR plus arch and turn it into an Image Factory Base Image object """ + """ + Take a build ID or NVR plus arch and turn it into + an Image Factory Base Image object + """ pim = PersistentImageManager.default_manager() build = self.session.getBuild(nvr) if not build: @@ -4471,7 +4570,8 @@ class BuildIndirectionImageTask(OzImageTask): tdl_full = os.path.join(builddir, build_tdl) if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)): - raise koji.BuildError("Missing TDL (%s) or qcow2 (%s) image for image (%s) - this should never happen" % + raise koji.BuildError("Missing TDL (%s) or qcow2 (%s) image for image (%s)" + " - this should never happen" % (build_tdl, build_diskimage, nvr)) # The sequence to recreate a valid persistent image is as follows @@ -4617,7 +4717,8 @@ class BuildIndirectionImageTask(OzImageTask): tlog.removeHandler(fhandler) self.uploadFile(ozlog) raise koji.ApplianceError('Image status is %s: %s' % - (target.target_image.status, target.target_image.status_detail)) + (target.target_image.status, + target.target_image.status_detail)) self.uploadFile(target.target_image.data, remoteName=os.path.basename(results_loc)) @@ -4644,7 +4745,8 @@ class BuildIndirectionImageTask(OzImageTask): # tag it if not opts.get('scratch') and not opts.get('skip_tag'): tag_task_id = self.session.host.subtask(method='tagBuild', - arglist=[target_info['dest_tag'], bld_info['id'], False, None, True], + arglist=[target_info['dest_tag'], + bld_info['id'], False, None, True], label='tag', parent=self.id, arch='noarch') self.wait(tag_task_id) @@ -4652,7 +4754,8 @@ class BuildIndirectionImageTask(OzImageTask): report = '' if opts.get('scratch'): respath = ', '.join( - [os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in [self.id]]) + [os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(tid)) for tid in [self.id]]) report += 'Scratch ' else: respath = koji.pathinfo.imagebuild(bld_info) @@ -4681,8 +4784,10 @@ class RebuildSRPM(BaseBuildTask): build_tag = self.session.getTag(build_tag, strict=True, event=event_id) rootopts = {'install_group': 'srpm-build', 'repo_id': repo_id} - br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id)) - broot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, **rootopts) + br_arch = self.find_arch('noarch', self.session.host.getHost( + ), self.session.getBuildConfig(build_tag['id'], event=event_id)) + broot = BuildRoot(self.session, self.options, + build_tag['id'], br_arch, self.id, **rootopts) broot.workdir = self.workdir self.logger.debug("Initializing buildroot") @@ -4720,7 +4825,8 @@ class RebuildSRPM(BaseBuildTask): release = koji.get_header_field(h, 'release') srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals() if srpm_name != os.path.basename(srpm): - raise koji.BuildError('srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))) + raise koji.BuildError('srpm name mismatch: %s != %s' % + (srpm_name, os.path.basename(srpm))) # upload srpm and return self.uploadFile(srpm) @@ -4784,12 +4890,15 @@ class BuildSRPMFromSCMTask(BaseBuildTask): rootopts = {'install_group': 'srpm-build', 'setup_dns': True, 'repo_id': repo_id} - if self.options.scm_credentials_dir is not None and os.path.isdir(self.options.scm_credentials_dir): + if self.options.scm_credentials_dir is not None and os.path.isdir( + self.options.scm_credentials_dir): rootopts['bind_opts'] = {'dirs': {self.options.scm_credentials_dir: '/credentials', }} # Force internal_dev_setup back to true because bind_opts is used to turn it off rootopts['internal_dev_setup'] = True - br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id)) - broot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, **rootopts) + br_arch = self.find_arch('noarch', self.session.host.getHost( + ), self.session.getBuildConfig(build_tag['id'], event=event_id)) + broot = BuildRoot(self.session, self.options, + build_tag['id'], br_arch, self.id, **rootopts) broot.workdir = self.workdir self.logger.debug("Initializing buildroot") @@ -4803,7 +4912,8 @@ class BuildSRPMFromSCMTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=opts.get('scratch')) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), + build_tag=build_tag, scratch=opts.get('scratch')) # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_callbacks("postSCMCheckout", @@ -4855,7 +4965,8 @@ class BuildSRPMFromSCMTask(BaseBuildTask): release = koji.get_header_field(h, 'release') srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals() if srpm_name != os.path.basename(srpm): - raise koji.BuildError('srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))) + raise koji.BuildError('srpm name mismatch: %s != %s' % + (srpm_name, os.path.basename(srpm))) # upload srpm and return self.uploadFile(srpm) @@ -4898,13 +5009,16 @@ Status: %(status)s\r %(failure_info)s\r """ - def handler(self, recipients, is_successful, tag_info, from_info, build_info, user_info, ignore_success=None, failure_msg=''): + def handler(self, recipients, is_successful, tag_info, from_info, + build_info, user_info, ignore_success=None, failure_msg=''): if len(recipients) == 0: self.logger.debug('task %i: no recipients, not sending notifications', self.id) return if ignore_success and is_successful: - self.logger.debug('task %i: tag operation successful and ignore success is true, not sending notifications', self.id) + self.logger.debug( + 'task %i: tag operation successful and ignore success is true, ' + 'not sending notifications', self.id) return build = self.session.getBuild(build_info) @@ -4972,7 +5086,8 @@ class BuildNotificationTask(BaseTaskHandler): _taskWeight = 0.1 # XXX externalize these templates somewhere - subject_templ = """Package: %(build_nvr)s Tag: %(dest_tag)s Status: %(status)s Built by: %(build_owner)s""" + subject_templ = "Package: %(build_nvr)s Tag: %(dest_tag)s Status: %(status)s " \ + "Built by: %(build_owner)s" message_templ = \ """From: %(from_addr)s\r Subject: %(subject)s\r @@ -5073,7 +5188,8 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r return build_pkg_name = build['package_name'] - build_pkg_evr = '%s%s-%s' % ((build['epoch'] and str(build['epoch']) + ':' or ''), build['version'], build['release']) + build_pkg_evr = '%s%s-%s' % ((build['epoch'] and str(build['epoch']) + + ':' or ''), build['version'], build['release']) build_nvr = koji.buildLabel(build) build_id = build['id'] build_owner = build['owner_name'] @@ -5099,7 +5215,9 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r cancel_info = "\r\nCanceled by: %s" % canceler['name'] elif build['state'] == koji.BUILD_STATES['FAILED']: failure_data = task_data[task_id]['result'] - failed_hosts = ['%s (%s)' % (task['host'], task['arch']) for task in task_data.values() if task['host'] and task['state'] == 'failed'] + failed_hosts = ['%s (%s)' % (task['host'], task['arch']) + for task in task_data.values() + if task['host'] and task['state'] == 'failed'] failure_info = "\r\n%s (%d) failed on %s:\r\n %s" % (build_nvr, build_id, ', '.join(failed_hosts), failure_data) @@ -5142,9 +5260,11 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r output += "logs:\r\n" for (file_, volume) in task['logs']: if tasks[task_state] != 'closed': - output += " %s/getfile?taskID=%s&name=%s&volume=%s\r\n" % (weburl, task['id'], file_, volume) + output += " %s/getfile?taskID=%s&name=%s&volume=%s\r\n" % ( + weburl, task['id'], file_, volume) else: - output += " %s\r\n" % '/'.join([buildurl, 'data', 'logs', task['build_arch'], file_]) + output += " %s\r\n" % '/'.join([buildurl, 'data', 'logs', + task['build_arch'], file_]) if task['rpms']: output += "rpms:\r\n" for file_ in task['rpms']: @@ -5152,11 +5272,13 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r if task['misc']: output += "misc:\r\n" for (file_, volume) in task['misc']: - output += " %s/getfile?taskID=%s&name=%s&volume=%s\r\n" % (weburl, task['id'], file_, volume) + output += " %s/getfile?taskID=%s&name=%s&volume=%s\r\n" % ( + weburl, task['id'], file_, volume) output += "\r\n" output += "\r\n" - changelog = koji.util.formatChangelog(self.session.getChangelogEntries(build_id, queryOpts={'limit': 3})).replace("\n", "\r\n") + changelog = koji.util.formatChangelog(self.session.getChangelogEntries( + build_id, queryOpts={'limit': 3})).replace("\n", "\r\n") if changelog: changelog = "Changelog:\r\n%s" % changelog @@ -5464,7 +5586,8 @@ class createDistRepoTask(BaseTaskHandler): "sparc": ("sparcv9v", "sparcv9", "sparcv8", "sparc", "noarch"), "sparc64": ("sparc64v", "sparc64", "noarch"), "alpha": ("alphaev6", "alphaev56", "alphaev5", "alpha", "noarch"), - "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", "noarch"), + "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", + "noarch"), "armhfp": ("armv7hl", "armv7hnl", "noarch"), "aarch64": ("aarch64", "noarch"), "riscv64": ("riscv64", "noarch"), @@ -5926,7 +6049,8 @@ enabled=1 for a in self.compat[arch]: # note: self.compat includes noarch for non-src already rpm_iter, builds = self.session.listTaggedRPMS(tag_id, - event=opts['event'], arch=a, latest=opts['latest'], + event=opts['event'], arch=a, + latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) for build in builds: builddirs[build['id']] = koji.pathinfo.build(build) @@ -6105,9 +6229,12 @@ class WaitrepoTask(BaseTaskHandler): repo = self.session.getRepo(taginfo['id']) if repo and repo != last_repo: if builds: - if koji.util.checkForBuilds(self.session, taginfo['id'], builds, repo['create_event']): - self.logger.debug("Successfully waited %s for %s to appear in the %s repo" % - (koji.util.duration(start), koji.util.printList(nvrs), taginfo['name'])) + if koji.util.checkForBuilds( + self.session, taginfo['id'], builds, repo['create_event']): + self.logger.debug("Successfully waited %s for %s to appear " + "in the %s repo" % + (koji.util.duration(start), koji.util.printList(nvrs), + taginfo['name'])) return repo elif newer_than: if repo['create_ts'] > newer_than: @@ -6120,8 +6247,10 @@ class WaitrepoTask(BaseTaskHandler): if (time.time() - start) > (self.TIMEOUT * 60.0): if builds: - raise koji.GenericError("Unsuccessfully waited %s for %s to appear in the %s repo" % - (koji.util.duration(start), koji.util.printList(nvrs), taginfo['name'])) + raise koji.GenericError("Unsuccessfully waited %s for %s to appear " + "in the %s repo" % + (koji.util.duration(start), koji.util.printList(nvrs), + taginfo['name'])) else: raise koji.GenericError("Unsuccessfully waited %s for a new %s repo" % (koji.util.duration(start), taginfo['name'])) diff --git a/builder/mergerepos b/builder/mergerepos index 1c02070..a6c5a35 100755 --- a/builder/mergerepos +++ b/builder/mergerepos @@ -61,7 +61,8 @@ MULTILIB_ARCHES = { def parse_args(args): """Parse our opts/args""" usage = """ - mergerepos: take 2 or more repositories and merge their metadata into a new repo using Koji semantics + mergerepos: take 2 or more repositories and merge their metadata into a new + repo using Koji semantics mergerepos --repo=url --repo=url --outputdir=/some/path""" @@ -74,7 +75,8 @@ def parse_args(args): parser.add_option("-a", "--arch", dest="arches", default=[], action="append", help="List of arches to include in the repo") parser.add_option("-b", "--blocked", default=None, - help="A file containing a list of srpm names to exclude from the merged repo") + help="A file containing a list of srpm names to exclude " + "from the merged repo") parser.add_option("--mode", default='koji', help="Select the merge mode") parser.add_option("-o", "--outputdir", default=None, help="Location to create the repository") @@ -175,18 +177,18 @@ class RepoMerge(object): For each package object, check if the srpm name has ever been seen before. If is has not, keep the package. If it has, check if the srpm name was first seen in the same repo as the current package. If so, keep the package from the srpm with the - highest NVR. If not, keep the packages from the first srpm we found, and delete packages from - all other srpms. + highest NVR. If not, keep the packages from the first srpm we found, and delete packages + from all other srpms. Packages with matching NVRs in multiple repos will be taken from the first repo. If the srpm name appears in the blocked package list, any packages generated from the srpm will be deleted from the package sack as well. - This method will also generate a file called "pkgorigins" and add it to the repo metadata. This - is a tab-separated map of package E:N-V-R.A to repo URL (as specified on the command-line). This - allows a package to be tracked back to its origin, even if the location field in the repodata does - not match the original repo location. + This method will also generate a file called "pkgorigins" and add it to the repo metadata. + This is a tab-separated map of package E:N-V-R.A to repo URL (as specified on the + command-line). This allows a package to be tracked back to its origin, even if the location + field in the repodata does not match the original repo location. """ if self.mode == 'simple': @@ -208,7 +210,8 @@ class RepoMerge(object): # to be using relative urls # XXX - kind of a hack, but yum leaves us little choice # force the pkg object to report a relative location - loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) + loc = """\n""" % yum.misc.to_xml(pkg.remote_path, + attrib=True) pkg._return_remote_location = make_const_func(loc) if pkg.sourcerpm in seen_srpms: # we're just looking at sourcerpms this pass and we've @@ -299,7 +302,8 @@ class RepoMerge(object): # to be using relative urls # XXX - kind of a hack, but yum leaves us little choice # force the pkg object to report a relative location - loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) + loc = """\n""" % yum.misc.to_xml(pkg.remote_path, + attrib=True) pkg._return_remote_location = make_const_func(loc) pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') diff --git a/cli/koji b/cli/koji index 545f3be..c450074 100755 --- a/cli/koji +++ b/cli/koji @@ -129,7 +129,8 @@ def get_options(): help=_("do not authenticate")) parser.add_option("--force-auth", action="store_true", default=False, help=_("authenticate even for read-only operations")) - parser.add_option("--authtype", help=_("force use of a type of authentication, options: noauth, ssl, password, or kerberos")) + parser.add_option("--authtype", help=_("force use of a type of authentication, options: " + "noauth, ssl, password, or kerberos")) parser.add_option("-d", "--debug", action="store_true", help=_("show debug output")) parser.add_option("--debug-xmlrpc", action="store_true", @@ -145,7 +146,8 @@ def get_options(): parser.add_option("--pkgurl", help=SUPPRESS_HELP) parser.add_option("--plugin-paths", metavar='PATHS', help=_("specify additional plugin paths (colon separated)")) - parser.add_option("--help-commands", action="store_true", default=False, help=_("list commands")) + parser.add_option("--help-commands", action="store_true", default=False, + help=_("list commands")) (options, args) = parser.parse_args() # load local config diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index f3a9cbf..b591e97 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -185,7 +185,8 @@ def handle_add_host(goptions, session, args): "[admin] Add a host" usage = _("usage: %prog add-host [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--krb-principal", help=_("set a non-default kerberos principal for the host")) + parser.add_option("--krb-principal", + help=_("set a non-default kerberos principal for the host")) (options, args) = parser.parse_args(args) if len(args) < 2: parser.error(_("Please specify a hostname and at least one arch")) @@ -208,7 +209,8 @@ def handle_edit_host(options, session, args): "[admin] Edit a host" usage = _("usage: %prog edit-host [ ...] [options]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--arches", help=_("Space or comma-separated list of supported architectures")) + parser.add_option("--arches", + help=_("Space or comma-separated list of supported architectures")) parser.add_option("--capacity", type="float", help=_("Capacity of this host")) parser.add_option("--description", metavar="DESC", help=_("Description of this host")) parser.add_option("--comment", help=_("A brief comment about this host")) @@ -358,7 +360,8 @@ def handle_add_pkg(goptions, session, args): if dsttag is None: print("No such tag: %s" % tag) sys.exit(1) - pkglist = dict([(p['package_name'], p['package_id']) for p in session.listPackages(tagID=dsttag['id'])]) + pkglist = dict([(p['package_name'], p['package_id']) + for p in session.listPackages(tagID=dsttag['id'])]) to_add = [] for package in args[1:]: package_id = pkglist.get(package, None) @@ -381,7 +384,8 @@ def handle_block_pkg(goptions, session, args): "[admin] Block a package in the listing for tag" usage = _("usage: %prog block-pkg [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--force", action='store_true', default=False, help=_("Override blocks and owner if necessary")) + parser.add_option("--force", action='store_true', default=False, + help=_("Override blocks and owner if necessary")) (options, args) = parser.parse_args(args) if len(args) < 2: parser.error(_("Please specify a tag and at least one package")) @@ -392,7 +396,8 @@ def handle_block_pkg(goptions, session, args): if dsttag is None: print("No such tag: %s" % tag) return 1 - pkglist = dict([(p['package_name'], p['package_id']) for p in session.listPackages(tagID=dsttag['id'], inherited=True)]) + pkglist = dict([(p['package_name'], p['package_id']) + for p in session.listPackages(tagID=dsttag['id'], inherited=True)]) ret = 0 for package in args[1:]: package_id = pkglist.get(package, None) @@ -429,7 +434,8 @@ def handle_remove_pkg(goptions, session, args): if dsttag is None: print("No such tag: %s" % tag) return 1 - pkglist = dict([(p['package_name'], p['package_id']) for p in session.listPackages(tagID=dsttag['id'])]) + pkglist = dict([(p['package_name'], p['package_id']) + for p in session.listPackages(tagID=dsttag['id'])]) ret = 0 for package in args[1:]: package_id = pkglist.get(package, None) @@ -472,7 +478,8 @@ def handle_build(options, session, args): help=_("Run the build at a lower priority")) (build_opts, args) = parser.parse_args(args) if len(args) != 2: - parser.error(_("Exactly two arguments (a build target and a SCM URL or srpm file) are required")) + parser.error(_("Exactly two arguments (a build target and a SCM URL or srpm file) are " + "required")) if build_opts.arch_override and not build_opts.scratch: parser.error(_("--arch_override is only allowed for --scratch builds")) activate_session(session, options) @@ -552,8 +559,10 @@ def handle_chain_build(options, session, args): # check that the destination tag is in the inheritance tree of the build tag # otherwise there is no way that a chain-build can work ancestors = session.getFullInheritance(build_target['build_tag']) - if dest_tag['id'] not in [build_target['build_tag']] + [ancestor['parent_id'] for ancestor in ancestors]: - print(_("Packages in destination tag %(dest_tag_name)s are not inherited by build tag %(build_tag_name)s" % build_target)) + if dest_tag['id'] not in [build_target['build_tag']] + \ + [ancestor['parent_id'] for ancestor in ancestors]: + print(_("Packages in destination tag %(dest_tag_name)s are not inherited by build tag " + "%(build_tag_name)s" % build_target)) print(_("Target %s is not usable for a chain-build" % build_target['name'])) return 1 @@ -582,7 +591,8 @@ def handle_chain_build(options, session, args): src_list.append(build_level) if len(src_list) < 2: - parser.error(_('You must specify at least one dependency between builds with : (colon)\nIf there are no dependencies, use the build command instead')) + parser.error(_('You must specify at least one dependency between builds with : (colon)\n' + 'If there are no dependencies, use the build command instead')) priority = None if build_opts.background: @@ -607,7 +617,8 @@ def handle_maven_build(options, session, args): usage += _("\n %prog maven-build --ini=CONFIG... [options] ") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--patches", action="store", metavar="URL", - help=_("SCM URL of a directory containing patches to apply to the sources before building")) + help=_("SCM URL of a directory containing patches to apply to the sources " + "before building")) parser.add_option("-G", "--goal", action="append", dest="goals", metavar="GOAL", default=[], help=_("Additional goal to run before \"deploy\"")) @@ -673,7 +684,8 @@ def handle_maven_build(options, session, args): parser.error(e.args[0]) opts = to_list(params.values())[0] if opts.pop('type', 'maven') != 'maven': - parser.error(_("Section %s does not contain a maven-build config") % to_list(params.keys())[0]) + parser.error(_("Section %s does not contain a maven-build config") % + to_list(params.keys())[0]) source = opts.pop('scmurl') else: source = args[1] @@ -704,16 +716,19 @@ def handle_wrapper_rpm(options, session, args): """[build] Build wrapper rpms for any archives associated with a build.""" usage = _("usage: %prog wrapper-rpm [options] ") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--create-build", action="store_true", help=_("Create a new build to contain wrapper rpms")) + parser.add_option("--create-build", action="store_true", + help=_("Create a new build to contain wrapper rpms")) parser.add_option("--ini", action="append", dest="inis", metavar="CONFIG", default=[], help=_("Pass build parameters via a .ini file")) parser.add_option("-s", "--section", help=_("Get build parameters from this section of the .ini")) - parser.add_option("--skip-tag", action="store_true", help=_("If creating a new build, don't tag it")) + parser.add_option("--skip-tag", action="store_true", + help=_("If creating a new build, don't tag it")) parser.add_option("--scratch", action="store_true", help=_("Perform a scratch build")) parser.add_option("--nowait", action="store_true", help=_("Don't wait on build")) - parser.add_option("--background", action="store_true", help=_("Run the build at a lower priority")) + parser.add_option("--background", action="store_true", + help=_("Run the build at a lower priority")) (build_opts, args) = parser.parse_args(args) if build_opts.inis: @@ -721,7 +736,8 @@ def handle_wrapper_rpm(options, session, args): parser.error(_("Exactly one argument (a build target) is required")) else: if len(args) < 3: - parser.error(_("You must provide a build target, a build ID or NVR, and a SCM URL to a specfile fragment")) + parser.error(_("You must provide a build target, a build ID or NVR, " + "and a SCM URL to a specfile fragment")) activate_session(session, options) target = args[0] @@ -733,7 +749,8 @@ def handle_wrapper_rpm(options, session, args): parser.error(e.args[0]) opts = to_list(params.values())[0] if opts.get('type') != 'wrapper': - parser.error(_("Section %s does not contain a wrapper-rpm config") % to_list(params.keys())[0]) + parser.error(_("Section %s does not contain a wrapper-rpm config") % + to_list(params.keys())[0]) url = opts['scmurl'] package = opts['buildrequires'][0] target_info = session.getBuildTarget(target, strict=True) @@ -852,7 +869,8 @@ def handle_call(goptions, session, args): usage = _("usage: %prog call [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--python", action="store_true", help=_("Use python syntax for values")) - parser.add_option("--kwargs", help=_("Specify keyword arguments as a dictionary (implies --python)")) + parser.add_option("--kwargs", + help=_("Specify keyword arguments as a dictionary (implies --python)")) parser.add_option("--json-output", action="store_true", help=_("Use JSON syntax for output")) (options, args) = parser.parse_args(args) if len(args) < 1: @@ -895,7 +913,8 @@ def anon_handle_mock_config(goptions, session, args): parser.add_option("--target", help=_("Create a mock config for a build target")) parser.add_option("--task", help=_("Duplicate the mock config of a previous task")) parser.add_option("--latest", action="store_true", help=_("use the latest redirect url")) - parser.add_option("--buildroot", help=_("Duplicate the mock config for the specified buildroot id")) + parser.add_option("--buildroot", + help=_("Duplicate the mock config for the specified buildroot id")) parser.add_option("--mockdir", default="/var/lib/mock", metavar="DIR", help=_("Specify mockdir")) parser.add_option("--topdir", metavar="DIR", @@ -1136,9 +1155,11 @@ def handle_import(goptions, session, args): "[admin] Import externally built RPMs into the database" usage = _("usage: %prog import [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading")) + parser.add_option("--link", action="store_true", + help=_("Attempt to hardlink instead of uploading")) parser.add_option("--test", action="store_true", help=_("Don't actually import")) - parser.add_option("--create-build", action="store_true", help=_("Auto-create builds as needed")) + parser.add_option("--create-build", action="store_true", + help=_("Auto-create builds as needed")) parser.add_option("--src-epoch", help=_("When auto-creating builds, use this epoch")) (options, args) = parser.parse_args(args) if len(args) < 1: @@ -1284,7 +1305,8 @@ def handle_import_cg(goptions, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--noprogress", action="store_true", help=_("Do not display progress of the upload")) - parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading")) + parser.add_option("--link", action="store_true", + help=_("Attempt to hardlink instead of uploading")) parser.add_option("--test", action="store_true", help=_("Don't actually import")) parser.add_option("--token", action="store", default=None, help=_("Build reservation token")) (options, args) = parser.parse_args(args) @@ -1443,7 +1465,8 @@ def handle_import_sig(goptions, session, args): parser.error(_("No such file: %s") % path) activate_session(session, goptions) for path in args: - data = koji.get_header_fields(path, ('name', 'version', 'release', 'arch', 'siggpg', 'sigpgp', 'sourcepackage')) + data = koji.get_header_fields(path, ('name', 'version', 'release', 'arch', 'siggpg', + 'sigpgp', 'sourcepackage')) if data['sourcepackage']: data['arch'] = 'src' sigkey = data['siggpg'] @@ -1463,7 +1486,8 @@ def handle_import_sig(goptions, session, args): print("No such rpm in system: %(name)s-%(version)s-%(release)s.%(arch)s" % data) continue if rinfo.get('external_repo_id'): - print("Skipping external rpm: %(name)s-%(version)s-%(release)s.%(arch)s@%(external_repo_name)s" % rinfo) + print("Skipping external rpm: %(name)s-%(version)s-%(release)s.%(arch)s@" + "%(external_repo_name)s" % rinfo) continue sighdr = koji.rip_rpm_sighdr(path) previous = session.queryRPMSigs(rpm_id=rinfo['id'], sigkey=sigkey) @@ -1490,7 +1514,8 @@ def handle_write_signed_rpm(goptions, session, args): "[admin] Write signed RPMs to disk" usage = _("usage: %prog write-signed-rpm [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--all", action="store_true", help=_("Write out all RPMs signed with this key")) + parser.add_option("--all", action="store_true", + help=_("Write out all RPMs signed with this key")) parser.add_option("--buildid", help=_("Specify a build id rather than an n-v-r")) (options, args) = parser.parse_args(args) if len(args) < 1: @@ -1703,7 +1728,8 @@ def handle_prune_signed_copies(options, session, args): # we were still tagged here sometime before the cutoff if options.debug: print("Build %s had protected tag %s until %s" - % (nvr, tag_name, time.asctime(time.localtime(our_entry['revoke_ts'])))) + % (nvr, tag_name, + time.asctime(time.localtime(our_entry['revoke_ts'])))) is_protected = True break replaced_ts = None @@ -2062,7 +2088,8 @@ def handle_list_signed(goptions, session, args): rinfo = session.getRPM(rpm_info, strict=True) rpm_idx[rinfo['id']] = rinfo if rinfo.get('external_repo_id'): - parser.error(_("External rpm: %(name)s-%(version)s-%(release)s.%(arch)s@%(external_repo_name)s") % rinfo) + parser.error(_("External rpm: %(name)s-%(version)s-%(release)s.%(arch)s@" + "%(external_repo_name)s") % rinfo) qopts['rpm_id'] = rinfo['id'] if options.build: build = options.build @@ -2123,13 +2150,18 @@ def handle_import_archive(options, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--noprogress", action="store_true", help=_("Do not display progress of the upload")) - parser.add_option("--create-build", action="store_true", help=_("Auto-create builds as needed")) - parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading")) - parser.add_option("--type", help=_("The type of archive being imported. Currently supported types: maven, win, image")) - parser.add_option("--type-info", help=_("Type-specific information to associate with the archives. " - "For Maven archives this should be a local path to a .pom file. " - "For Windows archives this should be relpath:platforms[:flags])) " - "Images need an arch")) + parser.add_option("--create-build", action="store_true", + help=_("Auto-create builds as needed")) + parser.add_option("--link", action="store_true", + help=_("Attempt to hardlink instead of uploading")) + parser.add_option("--type", + help=_("The type of archive being imported. " + "Currently supported types: maven, win, image")) + parser.add_option("--type-info", + help=_("Type-specific information to associate with the archives. " + "For Maven archives this should be a local path to a .pom file. " + "For Windows archives this should be relpath:platforms[:flags])) " + "Images need an arch")) (suboptions, args) = parser.parse_args(args) if not len(args) > 1: @@ -2297,11 +2329,14 @@ def anon_handle_latest_build(goptions, session, args): usage = _("usage: %prog latest-build [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--arch", help=_("List all of the latest packages for this arch")) - parser.add_option("--all", action="store_true", help=_("List all of the latest packages for this tag")) + parser.add_option("--all", action="store_true", + help=_("List all of the latest packages for this tag")) parser.add_option("--quiet", action="store_true", default=goptions.quiet, help=_("Do not print the header information")) parser.add_option("--paths", action="store_true", help=_("Show the file paths")) - parser.add_option("--type", help=_("Show builds of the given type only. Currently supported types: maven")) + parser.add_option("--type", + help=_("Show builds of the given type only. " + "Currently supported types: maven")) (options, args) = parser.parse_args(args) if len(args) == 0: parser.error(_("A tag name must be specified")) @@ -2338,20 +2373,24 @@ def anon_handle_latest_build(goptions, session, args): if options.type == 'maven': for x in data: x['path'] = pathinfo.mavenbuild(x) - fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s %(owner_name)s" + fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s " \ + "%(maven_artifact_id)-20s %(owner_name)s" else: for x in data: x['path'] = pathinfo.build(x) fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s" else: if options.type == 'maven': - fmt = "%(nvr)-40s %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s %(owner_name)s" + fmt = "%(nvr)-40s %(tag_name)-20s %(maven_group_id)-20s " \ + "%(maven_artifact_id)-20s %(owner_name)s" else: fmt = "%(nvr)-40s %(tag_name)-20s %(owner_name)s" if not options.quiet: if options.type == 'maven': - print("%-40s %-20s %-20s %-20s %s" % ("Build", "Tag", "Group Id", "Artifact Id", "Built by")) - print("%s %s %s %s %s" % ("-" * 40, "-" * 20, "-" * 20, "-" * 20, "-" * 16)) + print("%-40s %-20s %-20s %-20s %s" % + ("Build", "Tag", "Group Id", "Artifact Id", "Built by")) + print("%s %s %s %s %s" % + ("-" * 40, "-" * 20, "-" * 20, "-" * 20, "-" * 16)) else: print("%-40s %-20s %s" % ("Build", "Tag", "Built by")) print("%s %s %s" % ("-" * 40, "-" * 20, "-" * 16)) @@ -2397,14 +2436,18 @@ def anon_handle_list_tagged(goptions, session, args): parser.add_option("--rpms", action="store_true", help=_("Show rpms instead of builds")) parser.add_option("--inherit", action="store_true", help=_("Follow inheritance")) parser.add_option("--latest", action="store_true", help=_("Only show the latest builds/rpms")) - parser.add_option("--latest-n", type='int', metavar="N", help=_("Only show the latest N builds/rpms")) + parser.add_option("--latest-n", type='int', metavar="N", + help=_("Only show the latest N builds/rpms")) parser.add_option("--quiet", action="store_true", default=goptions.quiet, help=_("Do not print the header information")) parser.add_option("--paths", action="store_true", help=_("Show the file paths")) parser.add_option("--sigs", action="store_true", help=_("Show signatures")) - parser.add_option("--type", help=_("Show builds of the given type only. Currently supported types: maven, win, image")) + parser.add_option("--type", + help=_("Show builds of the given type only. " + "Currently supported types: maven, win, image")) parser.add_option("--event", type='int', metavar="EVENT#", help=_("query at event")) - parser.add_option("--ts", type='int', metavar="TIMESTAMP", help=_("query at last event before timestamp")) + parser.add_option("--ts", type='int', metavar="TIMESTAMP", + help=_("query at last event before timestamp")) parser.add_option("--repo", type='int', metavar="REPO#", help=_("query at event for a repo")) (options, args) = parser.parse_args(args) if len(args) == 0: @@ -2473,20 +2516,24 @@ def anon_handle_list_tagged(goptions, session, args): if options.type == 'maven': for x in data: x['path'] = pathinfo.mavenbuild(x) - fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s %(owner_name)s" + fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s " \ + "%(maven_artifact_id)-20s %(owner_name)s" else: for x in data: x['path'] = pathinfo.build(x) fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s" else: if options.type == 'maven': - fmt = "%(nvr)-40s %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s %(owner_name)s" + fmt = "%(nvr)-40s %(tag_name)-20s %(maven_group_id)-20s " \ + "%(maven_artifact_id)-20s %(owner_name)s" else: fmt = "%(nvr)-40s %(tag_name)-20s %(owner_name)s" if not options.quiet: if options.type == 'maven': - print("%-40s %-20s %-20s %-20s %s" % ("Build", "Tag", "Group Id", "Artifact Id", "Built by")) - print("%s %s %s %s %s" % ("-" * 40, "-" * 20, "-" * 20, "-" * 20, "-" * 16)) + print("%-40s %-20s %-20s %-20s %s" % + ("Build", "Tag", "Group Id", "Artifact Id", "Built by")) + print("%s %s %s %s %s" % + ("-" * 40, "-" * 20, "-" * 20, "-" * 20, "-" * 16)) else: print("%-40s %-20s %s" % ("Build", "Tag", "Built by")) print("%s %s %s" % ("-" * 40, "-" * 20, "-" * 16)) @@ -2756,7 +2803,8 @@ def anon_handle_list_channels(goptions, session, args): if not options.quiet: print('Channel Enabled Ready Disbld Load Cap Perc') for channel in channels: - print("%(name)-15s %(enabled)6d %(ready)6d %(disabled)6d %(load)6d %(capacity)6d %(perc_load)6d%%" % channel) + print("%(name)-15s %(enabled)6d %(ready)6d %(disabled)6d %(load)6d %(capacity)6d " + "%(perc_load)6d%%" % channel) def anon_handle_list_hosts(goptions, session, args): @@ -2766,10 +2814,13 @@ def anon_handle_list_hosts(goptions, session, args): parser.add_option("--arch", action="append", default=[], help=_("Specify an architecture")) parser.add_option("--channel", help=_("Specify a channel")) parser.add_option("--ready", action="store_true", help=_("Limit to ready hosts")) - parser.add_option("--not-ready", action="store_false", dest="ready", help=_("Limit to not ready hosts")) + parser.add_option("--not-ready", action="store_false", dest="ready", + help=_("Limit to not ready hosts")) parser.add_option("--enabled", action="store_true", help=_("Limit to enabled hosts")) - parser.add_option("--not-enabled", action="store_false", dest="enabled", help=_("Limit to not enabled hosts")) - parser.add_option("--disabled", action="store_false", dest="enabled", help=_("Alias for --not-enabled")) + parser.add_option("--not-enabled", action="store_false", dest="enabled", + help=_("Limit to not enabled hosts")) + parser.add_option("--disabled", action="store_false", dest="enabled", + help=_("Alias for --not-enabled")) parser.add_option("--quiet", action="store_true", default=goptions.quiet, help=_("Do not print header information")) parser.add_option("--show-channels", action="store_true", help=_("Show host's channels")) @@ -2824,11 +2875,13 @@ def anon_handle_list_hosts(goptions, session, args): else: longest_host = 8 if not options.quiet: - hdr = "{hostname:<{longest_host}} Enb Rdy Load/Cap Arches Last Update".format(longest_host=longest_host, hostname='Hostname') + hdr = "{hostname:<{longest_host}} Enb Rdy Load/Cap Arches Last Update".format( + longest_host=longest_host, hostname='Hostname') if options.show_channels: hdr += " Channels" print(hdr) - mask = "%%(name)-%ss %%(enabled)-3s %%(ready)-3s %%(task_load)4.1f/%%(capacity)-4.1f %%(arches)-16s %%(update)-19s" % longest_host + mask = "%%(name)-%ss %%(enabled)-3s %%(ready)-3s %%(task_load)4.1f/%%(capacity)-4.1f " \ + "%%(arches)-16s %%(update)-19s" % longest_host if options.show_channels: mask += " %(channels)s" for host in hosts: @@ -2848,7 +2901,8 @@ def anon_handle_list_pkgs(goptions, session, args): parser.add_option("--show-blocked", action="store_true", help=_("Show blocked packages")) parser.add_option("--show-dups", action="store_true", help=_("Show superseded owners")) parser.add_option("--event", type='int', metavar="EVENT#", help=_("query at event")) - parser.add_option("--ts", type='int', metavar="TIMESTAMP", help=_("query at last event before timestamp")) + parser.add_option("--ts", type='int', metavar="TIMESTAMP", + help=_("query at last event before timestamp")) parser.add_option("--repo", type='int', metavar="REPO#", help=_("query at event for a repo")) (options, args) = parser.parse_args(args) if len(args) != 0: @@ -3043,7 +3097,8 @@ def anon_handle_rpminfo(goptions, session, args): "[info] Print basic information about an RPM" usage = _("usage: %prog rpminfo [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--buildroots", action="store_true", help=_("show buildroots the rpm was used in")) + parser.add_option("--buildroots", action="store_true", + help=_("show buildroots the rpm was used in")) (options, args) = parser.parse_args(args) if len(args) < 1: parser.error(_("Please specify an RPM")) @@ -3071,10 +3126,13 @@ def anon_handle_rpminfo(goptions, session, args): print("External Repository: %(name)s [%(id)i]" % repo) print("External Repository url: %(url)s" % repo) else: - print("RPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info))) + print("RPM Path: %s" % + os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info))) print("SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo) - print("SRPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(buildinfo))) - print("Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(info['buildtime']))) + print("SRPM Path: %s" % + os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(buildinfo))) + print("Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', + time.localtime(info['buildtime']))) print("SIGMD5: %(payloadhash)s" % info) print("Size: %(size)s" % info) if not info.get('external_repo_id', 0): @@ -3087,7 +3145,8 @@ def anon_handle_rpminfo(goptions, session, args): else: br_info = session.getBuildroot(info['buildroot_id']) if br_info['br_type'] == koji.BR_TYPES['STANDARD']: - print("Buildroot: %(id)i (tag %(tag_name)s, arch %(arch)s, repo %(repo_id)i)" % br_info) + print("Buildroot: %(id)i (tag %(tag_name)s, arch %(arch)s, repo %(repo_id)i)" % + br_info) print("Build Host: %(host_name)s" % br_info) print("Build Task: %(task_id)i" % br_info) else: @@ -3110,7 +3169,8 @@ def anon_handle_buildinfo(goptions, session, args): "[info] Print basic information about a build" usage = _("usage: %prog buildinfo [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--changelog", action="store_true", help=_("Show the changelog for the build")) + parser.add_option("--changelog", action="store_true", + help=_("Show the changelog for the build")) (options, args) = parser.parse_args(args) if len(args) < 1: parser.error(_("Please specify a build")) @@ -3162,7 +3222,8 @@ def anon_handle_buildinfo(goptions, session, args): print("Maven archives:") for archive in maven_archives: archives_seen.setdefault(archive['id'], 1) - print(os.path.join(koji.pathinfo.mavenbuild(info), koji.pathinfo.mavenfile(archive))) + print(os.path.join(koji.pathinfo.mavenbuild(info), + koji.pathinfo.mavenfile(archive))) win_archives = session.listArchives(buildID=info['id'], type='win') if win_archives: print("Windows archives:") @@ -3239,7 +3300,8 @@ def anon_handle_hostinfo(goptions, session, args): else: update = update[:update.find('.')] print("Last Update: %s" % update) - print("Channels: %s" % ' '.join([c['name'] for c in session.listChannels(hostID=info['id'])])) + print("Channels: %s" % ' '.join([c['name'] + for c in session.listChannels(hostID=info['id'])])) print("Active Buildroots:") states = {0: "INIT", 1: "WAITING", 2: "BUILDING"} rows = [('NAME', 'STATE', 'CREATION TIME')] @@ -3317,7 +3379,8 @@ def handle_clone_tag(goptions, session, args): dsttag = session.getTag(args[1]) if not srctag: parser.error(_("Unknown src-tag: %s" % args[0])) - if (srctag['locked'] and not options.force) or (dsttag and dsttag['locked'] and not options.force): + if (srctag['locked'] and not options.force) \ + or (dsttag and dsttag['locked'] and not options.force): parser.error(_("Error: You are attempting to clone from or to a tag which is locked.\n" "Please use --force if this is what you really want to do.")) @@ -3721,7 +3784,8 @@ def handle_clone_tag(goptions, session, args): for changes in chgpkglist: sys.stdout.write(pfmt % changes) sys.stdout.write('\n') - sys.stdout.write(bfmt % ('Action', 'From/To Package', 'Build(s)', 'State', 'Owner', 'From Tag')) + sys.stdout.write(bfmt % + ('Action', 'From/To Package', 'Build(s)', 'State', 'Owner', 'From Tag')) sys.stdout.write(bfmt % ('-' * 7, '-' * 28, '-' * 40, '-' * 10, '-' * 10, '-' * 10)) for changes in chgbldlist: sys.stdout.write(bfmt % changes) @@ -3808,7 +3872,8 @@ def handle_edit_target(goptions, session, args): return 1 targetInfo['dest_tag_name'] = options.dest_tag - session.editBuildTarget(targetInfo['orig_name'], targetInfo['name'], targetInfo['build_tag_name'], targetInfo['dest_tag_name']) + session.editBuildTarget(targetInfo['orig_name'], targetInfo['name'], + targetInfo['build_tag_name'], targetInfo['dest_tag_name']) def handle_remove_target(goptions, session, args): @@ -3924,11 +3989,13 @@ def anon_handle_list_tag_inheritance(goptions, session, args): "[info] Print the inheritance information for a tag" usage = _("usage: %prog list-tag-inheritance [options] ") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--reverse", action="store_true", help=_("Process tag's children instead of its parents")) + parser.add_option("--reverse", action="store_true", + help=_("Process tag's children instead of its parents")) parser.add_option("--stop", help=_("Stop processing inheritance at this tag")) parser.add_option("--jump", help=_("Jump from one tag to another when processing inheritance")) parser.add_option("--event", type='int', metavar="EVENT#", help=_("query at event")) - parser.add_option("--ts", type='int', metavar="TIMESTAMP", help=_("query at last event before timestamp")) + parser.add_option("--ts", type='int', metavar="TIMESTAMP", + help=_("query at last event before timestamp")) parser.add_option("--repo", type='int', metavar="REPO#", help=_("query at event for a repo")) (options, args) = parser.parse_args(args) if len(args) != 1: @@ -4036,7 +4103,8 @@ def anon_handle_list_tag_history(goptions, session, args): parser.add_option("--build", help=_("Only show data for a specific build")) parser.add_option("--package", help=_("Only show data for a specific package")) parser.add_option("--tag", help=_("Only show data for a specific tag")) - parser.add_option("--all", action="store_true", help=_("Allows listing the entire global history")) + parser.add_option("--all", action="store_true", + help=_("Allows listing the entire global history")) (options, args) = parser.parse_args(args) koji.util.deprecated("list-tag-history is deprecated and will be removed in a future version. " "See: https://pagure.io/koji/issue/836") @@ -4109,7 +4177,8 @@ def _print_histline(entry, **kwargs): if event_id != other[0]: bad_edit = "non-matching" if bad_edit: - print("Warning: unusual edit at event %i in table %s (%s)" % (event_id, table, bad_edit)) + print("Warning: unusual edit at event %i in table %s (%s)" % + (event_id, table, bad_edit)) # we'll simply treat them as separate events pprint.pprint(entry) pprint.pprint(edit) @@ -4333,28 +4402,36 @@ def anon_handle_list_history(goptions, session, args): parser.add_option("--build", help=_("Only show data for a specific build")) parser.add_option("--package", help=_("Only show data for a specific package")) parser.add_option("--tag", help=_("Only show data for a specific tag")) - parser.add_option("--editor", "--by", metavar="USER", help=_("Only show entries modified by user")) + parser.add_option("--editor", "--by", metavar="USER", + help=_("Only show entries modified by user")) parser.add_option("--user", help=_("Only show entries affecting a user")) parser.add_option("--permission", help=_("Only show entries relating to a given permission")) parser.add_option("--cg", help=_("Only show entries relating to a given permission")) - parser.add_option("--external-repo", "--erepo", help=_("Only show entries relating to a given external repo")) - parser.add_option("--build-target", "--target", help=_("Only show entries relating to a given build target")) + parser.add_option("--external-repo", "--erepo", + help=_("Only show entries relating to a given external repo")) + parser.add_option("--build-target", "--target", + help=_("Only show entries relating to a given build target")) parser.add_option("--group", help=_("Only show entries relating to a given group")) parser.add_option("--host", help=_("Only show entries related to given host")) parser.add_option("--channel", help=_("Only show entries related to given channel")) - parser.add_option("--before", metavar="TIMESTAMP", help=_("Only show entries before timestamp")) + parser.add_option("--before", metavar="TIMESTAMP", + help=_("Only show entries before timestamp")) parser.add_option("--after", metavar="TIMESTAMP", help=_("Only show entries after timestamp")) - parser.add_option("--before-event", metavar="EVENT_ID", type='int', help=_("Only show entries before event")) - parser.add_option("--after-event", metavar="EVENT_ID", type='int', help=_("Only show entries after event")) + parser.add_option("--before-event", metavar="EVENT_ID", type='int', + help=_("Only show entries before event")) + parser.add_option("--after-event", metavar="EVENT_ID", type='int', + help=_("Only show entries after event")) parser.add_option("--watch", action="store_true", help=_("Monitor history data")) - parser.add_option("--active", action='store_true', help=_("Only show entries that are currently active")) + parser.add_option("--active", action='store_true', + help=_("Only show entries that are currently active")) parser.add_option("--revoked", action='store_false', dest='active', help=_("Only show entries that are currently revoked")) parser.add_option("--context", action="store_true", help=_("Show related entries")) parser.add_option("-s", "--show", action="append", help=_("Show data from selected tables")) parser.add_option("-v", "--verbose", action="store_true", help=_("Show more detail")) parser.add_option("-e", "--events", action="store_true", help=_("Show event ids")) - parser.add_option("--all", action="store_true", help=_("Allows listing the entire global history")) + parser.add_option("--all", action="store_true", + help=_("Allows listing the entire global history")) (options, args) = parser.parse_args(args) if len(args) != 0: parser.error(_("This command takes no arguments")) @@ -4556,7 +4633,8 @@ def _do_parseTaskParams(session, method, task_id, topdir): if len(params) > 2: _handleOpts(lines, params[2]) elif method in ('createLiveCD', 'createAppliance', 'createLiveMedia'): - argnames = ['Name', 'Version', 'Release', 'Arch', 'Target Info', 'Build Tag', 'Repo', 'Kickstart File'] + argnames = ['Name', 'Version', 'Release', 'Arch', 'Target Info', 'Build Tag', 'Repo', + 'Kickstart File'] for n, v in zip(argnames, params): lines.append("%s: %s" % (n, v)) if len(params) > 8: @@ -4580,7 +4658,8 @@ def _do_parseTaskParams(session, method, task_id, topdir): lines.append("Old Repo ID: %i" % oldrepo['id']) lines.append("Old Repo Creation: %s" % koji.formatTimeLong(oldrepo['creation_time'])) if len(params) > 3: - lines.append("External Repos: %s" % ', '.join([ext['external_repo_name'] for ext in params[3]])) + lines.append("External Repos: %s" % + ', '.join([ext['external_repo_name'] for ext in params[3]])) elif method == 'tagNotification': destTag = session.getTag(params[2]) srcTag = None @@ -4604,7 +4683,8 @@ def _do_parseTaskParams(session, method, task_id, topdir): lines.append("Subtasks:") for subtask in params[1]: lines.append(" Method: %s" % subtask[0]) - lines.append(" Parameters: %s" % ", ".join([str(subparam) for subparam in subtask[1]])) + lines.append(" Parameters: %s" % + ", ".join([str(subparam) for subparam in subtask[1]])) if len(subtask) > 2 and subtask[2]: subopts = subtask[2] _handleOpts(lines, subopts, prefix=' ') @@ -4682,7 +4762,8 @@ def _printTaskInfo(session, task_id, topdir, level=0, recurse=True, verbose=True if buildroot_infos: print("%sBuildroots:" % indent) for root in buildroot_infos: - print("%s %s/%s-%d-%d/" % (indent, BUILDDIR, root['tag_name'], root['id'], root['repo_id'])) + print("%s %s/%s-%d-%d/" % + (indent, BUILDDIR, root['tag_name'], root['id'], root['repo_id'])) if logs: print("%sLog Files:" % indent) for log_path in logs: @@ -4707,7 +4788,8 @@ def anon_handle_taskinfo(goptions, session, args): """[info] Show information about a task""" usage = _("usage: %prog taskinfo [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("-r", "--recurse", action="store_true", help=_("Show children of this task as well")) + parser.add_option("-r", "--recurse", action="store_true", + help=_("Show children of this task as well")) parser.add_option("-v", "--verbose", action="store_true", help=_("Be verbose")) (options, args) = parser.parse_args(args) if len(args) < 1: @@ -4725,7 +4807,8 @@ def anon_handle_taginfo(goptions, session, args): usage = _("usage: %prog taginfo [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--event", type='int', metavar="EVENT#", help=_("query at event")) - parser.add_option("--ts", type='int', metavar="TIMESTAMP", help=_("query at last event before timestamp")) + parser.add_option("--ts", type='int', metavar="TIMESTAMP", + help=_("query at last event before timestamp")) parser.add_option("--repo", type='int', metavar="REPO#", help=_("query at event for a repo")) (options, args) = parser.parse_args(args) if len(args) < 1: @@ -4766,7 +4849,8 @@ def anon_handle_taginfo(goptions, session, args): print("Required permission: %r" % perms.get(perm_id, perm_id)) if session.mavenEnabled(): print("Maven support?: %s" % (info['maven_support'] and 'yes' or 'no')) - print("Include all Maven archives?: %s" % (info['maven_include_all'] and 'yes' or 'no')) + print("Include all Maven archives?: %s" % + (info['maven_include_all'] and 'yes' or 'no')) if 'extra' in info: print("Tag options:") for key in sorted(info['extra'].keys()): @@ -4788,7 +4872,8 @@ def anon_handle_taginfo(goptions, session, args): if event: print(" %s (%s)" % (target['name'], target['build_tag_name'])) else: - print(" %s (%s, %s)" % (target['name'], target['build_tag_name'], repos[target['build_tag']])) + print(" %s (%s, %s)" % + (target['name'], target['build_tag_name'], repos[target['build_tag']])) if build_targets: print("This tag is a buildroot for one or more targets") if not event: @@ -4817,8 +4902,10 @@ def handle_add_tag(goptions, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--parent", help=_("Specify parent")) parser.add_option("--arches", help=_("Specify arches")) - parser.add_option("--maven-support", action="store_true", help=_("Enable creation of Maven repos for this tag")) - parser.add_option("--include-all", action="store_true", help=_("Include all packages in this tag when generating Maven repos")) + parser.add_option("--maven-support", action="store_true", + help=_("Enable creation of Maven repos for this tag")) + parser.add_option("--include-all", action="store_true", + help=_("Include all packages in this tag when generating Maven repos")) parser.add_option("-x", "--extra", action="append", default=[], metavar="key=value", help=_("Set tag extra option")) (options, args) = parser.parse_args(args) @@ -4856,10 +4943,15 @@ def handle_edit_tag(goptions, session, args): parser.add_option("--lock", action="store_true", help=_("Lock the tag")) parser.add_option("--unlock", action="store_true", help=_("Unlock the tag")) parser.add_option("--rename", help=_("Rename the tag")) - parser.add_option("--maven-support", action="store_true", help=_("Enable creation of Maven repos for this tag")) - parser.add_option("--no-maven-support", action="store_true", help=_("Disable creation of Maven repos for this tag")) - parser.add_option("--include-all", action="store_true", help=_("Include all packages in this tag when generating Maven repos")) - parser.add_option("--no-include-all", action="store_true", help=_("Do not include all packages in this tag when generating Maven repos")) + parser.add_option("--maven-support", action="store_true", + help=_("Enable creation of Maven repos for this tag")) + parser.add_option("--no-maven-support", action="store_true", + help=_("Disable creation of Maven repos for this tag")) + parser.add_option("--include-all", action="store_true", + help=_("Include all packages in this tag when generating Maven repos")) + parser.add_option("--no-include-all", action="store_true", + help=_("Do not include all packages in this tag when generating Maven " + "repos")) parser.add_option("-x", "--extra", action="append", default=[], metavar="key=value", help=_("Set tag extra option")) parser.add_option("-r", "--remove-extra", action="append", default=[], metavar="key", @@ -4947,7 +5039,8 @@ def handle_lock_tag(goptions, session, args): print(_("Tag %s: %s permission already required") % (tag['name'], perm)) continue elif options.test: - print(_("Would have set permission requirement %s for tag %s") % (perm, tag['name'])) + print(_("Would have set permission requirement %s for tag %s") % + (perm, tag['name'])) continue session.editTag2(tag['id'], perm=perm_id) @@ -5008,7 +5101,8 @@ def handle_add_tag_inheritance(goptions, session, args): (options, args) = parser.parse_args(args) if len(args) != 2: - parser.error(_("This command takes exctly two argument: a tag name or ID and that tag's new parent name or ID")) + parser.error(_("This command takes exctly two argument: a tag name or ID and that tag's " + "new parent name or ID")) activate_session(session, goptions) @@ -5026,12 +5120,14 @@ def handle_add_tag_inheritance(goptions, session, args): samePriority = [datum for datum in inheritanceData if datum['priority'] == priority] if sameParents and not options.force: - print(_("Error: You are attempting to add %s as %s's parent even though it already is %s's parent.") + print(_("Error: You are attempting to add %s as %s's parent even though it already is " + "%s's parent.") % (parent['name'], tag['name'], tag['name'])) print(_("Please use --force if this is what you really want to do.")) return if samePriority: - print(_("Error: There is already an active inheritance with that priority on %s, please specify a different priority with --priority." % tag['name'])) + print(_("Error: There is already an active inheritance with that priority on %s, " + "please specify a different priority with --priority." % tag['name'])) return new_data = {} @@ -5064,7 +5160,8 @@ def handle_edit_tag_inheritance(goptions, session, args): parser.error(_("This command takes at least one argument: a tag name or ID")) if len(args) > 3: - parser.error(_("This command takes at most three argument: a tag name or ID, a parent tag name or ID, and a priority")) + parser.error(_("This command takes at most three argument: a tag name or ID, " + "a parent tag name or ID, and a priority")) activate_session(session, goptions) @@ -5107,7 +5204,8 @@ def handle_edit_tag_inheritance(goptions, session, args): inheritanceData = session.getInheritanceData(tag['id']) samePriority = [datum for datum in inheritanceData if datum['priority'] == options.priority] if samePriority: - print(_("Error: There is already an active inheritance with that priority on %s, please specify a different priority with --priority.") % tag['name']) + print(_("Error: There is already an active inheritance with that priority on %s, " + "please specify a different priority with --priority.") % tag['name']) return 1 new_data = data.copy() @@ -5144,7 +5242,8 @@ def handle_remove_tag_inheritance(goptions, session, args): parser.error(_("This command takes at least one argument: a tag name or ID")) if len(args) > 3: - parser.error(_("This command takes at most three argument: a tag name or ID, a parent tag name or ID, and a priority")) + parser.error(_("This command takes at most three argument: a tag name or ID, a parent tag " + "name or ID, and a priority")) activate_session(session, goptions) @@ -5203,7 +5302,8 @@ def anon_handle_show_groups(goptions, session, args): parser.add_option("-x", "--expand", action="store_true", default=False, help=_("Expand groups in comps format")) parser.add_option("--spec", action="store_true", help=_("Print build spec")) - parser.add_option("--show-blocked", action="store_true", dest="incl_blocked", help=_("Show blocked packages")) + parser.add_option("--show-blocked", action="store_true", dest="incl_blocked", + help=_("Show blocked packages")) (options, args) = parser.parse_args(args) if len(args) != 1: parser.error(_("Incorrect number of arguments")) @@ -5232,9 +5332,11 @@ def anon_handle_list_external_repos(goptions, session, args): parser.add_option("--id", type="int", help=_("Select by id")) parser.add_option("--tag", help=_("Select by tag")) parser.add_option("--used", action='store_true', help=_("List which tags use the repo(s)")) - parser.add_option("--inherit", action='store_true', help=_("Follow tag inheritance when selecting by tag")) + parser.add_option("--inherit", action='store_true', + help=_("Follow tag inheritance when selecting by tag")) parser.add_option("--event", type='int', metavar="EVENT#", help=_("Query at event")) - parser.add_option("--ts", type='int', metavar="TIMESTAMP", help=_("Query at last event before timestamp")) + parser.add_option("--ts", type='int', metavar="TIMESTAMP", + help=_("Query at last event before timestamp")) parser.add_option("--repo", type='int', metavar="REPO#", help=_("Query at event corresponding to (nonexternal) repo")) parser.add_option("--quiet", action="store_true", default=goptions.quiet, @@ -5413,7 +5515,8 @@ def handle_remove_external_repo(goptions, session, args): if delete: # removing entirely if current_tags and not options.force: - print(_("Error: external repo %s used by tag(s): %s") % (repo, ', '.join(current_tags))) + print(_("Error: external repo %s used by tag(s): %s") % + (repo, ', '.join(current_tags))) print(_("Use --force to remove anyway")) return 1 session.deleteExternalRepo(args[0]) @@ -5431,8 +5534,8 @@ def handle_spin_livecd(options, session, args): """[build] Create a live CD image given a kickstart file""" # Usage & option parsing. - usage = _("usage: %prog spin-livecd [options] " + - " ") + usage = _("usage: %prog spin-livecd [options] " + "") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--wait", action="store_true", help=_("Wait on the livecd creation, even if running in the background")) @@ -5449,8 +5552,8 @@ def handle_spin_livecd(options, session, args): parser.add_option("--scratch", action="store_true", help=_("Create a scratch LiveCD image")) parser.add_option("--repo", action="append", - help=_("Specify a repo that will override the repo used to install " + - "RPMs in the LiveCD. May be used multiple times. The " + + help=_("Specify a repo that will override the repo used to install " + "RPMs in the LiveCD. May be used multiple times. The " "build tag repo associated with the target is the default.")) parser.add_option("--release", help=_("Forcibly set the release field")) parser.add_option("--volid", help=_("Set the volume id")) @@ -5463,8 +5566,8 @@ def handle_spin_livecd(options, session, args): # Make sure the target and kickstart is specified. print('spin-livecd is deprecated and will be replaced with spin-livemedia') if len(args) != 5: - parser.error(_("Five arguments are required: a name, a version, an" + - " architecture, a build target, and a relative path to" + + parser.error(_("Five arguments are required: a name, a version, an" + " architecture, a build target, and a relative path to" " a kickstart file.")) if task_options.volid is not None and len(task_options.volid) > 32: parser.error(_('Volume ID has a maximum length of 32 characters')) @@ -5476,8 +5579,8 @@ def handle_spin_livemedia(options, session, args): """[build] Create a livemedia image given a kickstart file""" # Usage & option parsing. - usage = _("usage: %prog spin-livemedia [options] " + - " ") + usage = _("usage: %prog spin-livemedia [options] " + "") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--wait", action="store_true", help=_("Wait on the livemedia creation, even if running in the background")) @@ -5496,8 +5599,8 @@ def handle_spin_livemedia(options, session, args): parser.add_option("--scratch", action="store_true", help=_("Create a scratch LiveMedia image")) parser.add_option("--repo", action="append", - help=_("Specify a repo that will override the repo used to install " + - "RPMs in the LiveMedia. May be used multiple times. The " + + help=_("Specify a repo that will override the repo used to install " + "RPMs in the LiveMedia. May be used multiple times. The " "build tag repo associated with the target is the default.")) parser.add_option("--release", help=_("Forcibly set the release field")) parser.add_option("--volid", help=_("Set the volume id")) @@ -5507,7 +5610,8 @@ def handle_spin_livemedia(options, session, args): help=_("Do not attempt to tag package")) parser.add_option("--can-fail", action="store", dest="optional_arches", metavar="ARCH1,ARCH2,...", default="", - help=_("List of archs which are not blocking for build (separated by commas.")) + help=_("List of archs which are not blocking for build " + "(separated by commas.")) parser.add_option('--lorax_dir', metavar='DIR', help=_('The relative path to the lorax templates ' 'directory within the checkout of "lorax_url".')) @@ -5519,9 +5623,9 @@ def handle_spin_livemedia(options, session, args): # Make sure the target and kickstart is specified. if len(args) != 5: - parser.error(_("Five arguments are required: a name, a version, a" + - " build target, an architecture, and a relative path to" + - " a kickstart file.")) + parser.error(_("Five arguments are required: a name, a version, a " + "build target, an architecture, and a relative path to " + "a kickstart file.")) if task_options.lorax_url is not None and task_options.lorax_dir is None: parser.error(_('The "--lorax_url" option requires that "--lorax_dir" ' 'also be used.')) @@ -5536,8 +5640,8 @@ def handle_spin_appliance(options, session, args): """[build] Create an appliance given a kickstart file""" # Usage & option parsing - usage = _("usage: %prog spin-appliance [options] " + - " ") + usage = _("usage: %prog spin-appliance [options] " + "") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--wait", action="store_true", help=_("Wait on the appliance creation, even if running in the background")) @@ -5554,8 +5658,8 @@ def handle_spin_appliance(options, session, args): parser.add_option("--scratch", action="store_true", help=_("Create a scratch appliance")) parser.add_option("--repo", action="append", - help=_("Specify a repo that will override the repo used to install " + - "RPMs in the appliance. May be used multiple times. The " + + help=_("Specify a repo that will override the repo used to install " + "RPMs in the appliance. May be used multiple times. The " "build tag repo associated with the target is the default.")) parser.add_option("--release", help=_("Forcibly set the release field")) parser.add_option("--specfile", metavar="URL", @@ -5563,13 +5667,13 @@ def handle_spin_appliance(options, session, args): parser.add_option("--skip-tag", action="store_true", help=_("Do not attempt to tag package")) parser.add_option("--vmem", metavar="VMEM", default=None, - help=_("Set the amount of virtual memory in the appliance in MB, " + + help=_("Set the amount of virtual memory in the appliance in MB, " "default is 512")) parser.add_option("--vcpu", metavar="VCPU", default=None, - help=_("Set the number of virtual cpus in the appliance, " + + help=_("Set the number of virtual cpus in the appliance, " "default is 1")) parser.add_option("--format", metavar="DISK_FORMAT", default='raw', - help=_("Disk format, default is raw. Other options are qcow, " + + help=_("Disk format, default is raw. Other options are qcow, " "qcow2, and vmx.")) (task_options, args) = parser.parse_args(args) @@ -5577,20 +5681,20 @@ def handle_spin_appliance(options, session, args): # Make sure the target and kickstart is specified. print('spin-appliance is deprecated and will be replaced with image-build') if len(args) != 5: - parser.error(_("Five arguments are required: a name, a version, " + - "an architecture, a build target, and a relative path" + - " to a kickstart file.")) + parser.error(_("Five arguments are required: a name, a version, " + "an architecture, a build target, and a relative path " + "to a kickstart file.")) return _build_image(options, task_options, session, args, 'appliance') def handle_image_build_indirection(options, session, args): """[build] Create a disk image using other disk images via the Indirection plugin""" - usage = _("usage: %prog image-build-indirection [base_image] " + + usage = _("usage: %prog image-build-indirection [base_image] " "[utility_image] [indirection_build_template]") usage += _("\n %prog image-build --config \n") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--config", - help=_("Use a configuration file to define image-build options " + + help=_("Use a configuration file to define image-build options " "instead of command line options (they will be ignored).")) parser.add_option("--background", action="store_true", help=_("Run the image creation task at a lower priority")) @@ -5615,11 +5719,14 @@ def handle_image_build_indirection(options, session, args): parser.add_option("--utility-image-build", help=_("NVR or build ID of the utility image to be used")) parser.add_option("--indirection-template", - help=_("Name of the local file, or SCM file containing the template used to drive the indirection plugin")) + help=_("Name of the local file, or SCM file containing the template used to " + "drive the indirection plugin")) parser.add_option("--indirection-template-url", - help=_("SCM URL containing the template used to drive the indirection plugin")) + help=_("SCM URL containing the template used to drive the indirection " + "plugin")) parser.add_option("--results-loc", - help=_("Relative path inside the working space image where the results should be extracted from")) + help=_("Relative path inside the working space image where the results " + "should be extracted from")) parser.add_option("--scratch", action="store_true", help=_("Create a scratch image")) parser.add_option("--wait", action="store_true", @@ -5646,8 +5753,9 @@ def _build_image_indirection(options, task_opts, session, args): raise koji.GenericError(_("You must specify either a base-image task or build ID/NVR")) required_opts = ['name', 'version', 'arch', 'target', 'indirection_template', 'results_loc'] - optional_opts = ['indirection_template_url', 'scratch', 'utility_image_task', 'utility_image_build', - 'base_image_task', 'base_image_build', 'release', 'skip_tag'] + optional_opts = ['indirection_template_url', 'scratch', 'utility_image_task', + 'utility_image_build', 'base_image_task', 'base_image_build', 'release', + 'skip_tag'] missing = [] for opt in required_opts: @@ -5655,7 +5763,8 @@ def _build_image_indirection(options, task_opts, session, args): missing.append(opt) if len(missing) > 0: - print("Missing the following required options: %s" % ' '.join(['--%s' % o.replace('_', '-') for o in missing])) + print("Missing the following required options: %s" % + ' '.join(['--%s' % o.replace('_', '-') for o in missing])) raise koji.GenericError(_("Missing required options specified above")) activate_session(session, options) @@ -5690,7 +5799,8 @@ def _build_image_indirection(options, task_opts, session, args): if not task_opts.indirection_template_url: if not task_opts.scratch: # only scratch builds can omit indirection_template_url - raise koji.GenericError(_("Non-scratch builds must provide a URL for the indirection template")) + raise koji.GenericError( + _("Non-scratch builds must provide a URL for the indirection template")) templatefile = task_opts.indirection_template serverdir = unique_path('cli-image-indirection') session.uploadWrapper(templatefile, serverdir, callback=callback) @@ -5726,29 +5836,29 @@ def handle_image_build(options, session, args): 'vsphere-ova', 'vagrant-virtualbox', 'vagrant-libvirt', 'vagrant-vmware-fusion', 'vagrant-hyperv', 'docker', 'raw-xz', 'liveimg-squashfs', 'tar-gz') - usage = _("usage: %prog image-build [options] " + + usage = _("usage: %prog image-build [options] " " [ ...]") usage += _("\n %prog image-build --config \n") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--background", action="store_true", help=_("Run the image creation task at a lower priority")) parser.add_option("--config", - help=_("Use a configuration file to define image-build options " + + help=_("Use a configuration file to define image-build options " "instead of command line options (they will be ignored).")) parser.add_option("--disk-size", default=10, help=_("Set the disk device size in gigabytes")) parser.add_option("--distro", - help=_("specify the RPM based distribution the image will be based " + - "on with the format RHEL-X.Y, CentOS-X.Y, SL-X.Y, or Fedora-NN. " + - "The packages for the Distro you choose must have been built " + + help=_("specify the RPM based distribution the image will be based " + "on with the format RHEL-X.Y, CentOS-X.Y, SL-X.Y, or Fedora-NN. " + "The packages for the Distro you choose must have been built " "in this system.")) parser.add_option("--format", default=[], action="append", - help=_("Convert results to one or more formats " + - "(%s), this option may be used " % ', '.join(formats) + - "multiple times. By default, specifying this option will " + - "omit the raw disk image (which is 10G in size) from the " + - "build results. If you really want it included with converted " + - "images, pass in 'raw' as an option.")) + help=_("Convert results to one or more formats " + "(%s), this option may be used " + "multiple times. By default, specifying this option will " + "omit the raw disk image (which is 10G in size) from the " + "build results. If you really want it included with converted " + "images, pass in 'raw' as an option.") % ', '.join(formats)) parser.add_option("--kickstart", help=_("Path to a local kickstart file")) parser.add_option("--ksurl", metavar="SCMURL", help=_("The URL to the SCM containing the kickstart file")) @@ -5759,17 +5869,17 @@ def handle_image_build(options, session, args): parser.add_option("--nowait", action="store_false", dest="wait", help=_("Don't wait on image creation")) parser.add_option("--ova-option", action="append", - help=_("Override a value in the OVA description XML. Provide a value " + + help=_("Override a value in the OVA description XML. Provide a value " "in a name=value format, such as 'ovf_memory_mb=6144'")) parser.add_option("--factory-parameter", nargs=2, action="append", - help=_("Pass a parameter to Image Factory. The results are highly specific " + - "to the image format being created. This is a two argument parameter " + + help=_("Pass a parameter to Image Factory. The results are highly specific " + "to the image format being created. This is a two argument parameter " "that can be specified an arbitrary number of times. For example: " "--factory-parameter docker_cmd '[ \"/bin/echo Hello World\" ]'")) parser.add_option("--release", help=_("Forcibly set the release field")) parser.add_option("--repo", action="append", - help=_("Specify a repo that will override the repo used to install " + - "RPMs in the image. May be used multiple times. The " + + help=_("Specify a repo that will override the repo used to install " + "RPMs in the image. May be used multiple times. The " "build tag repo associated with the target is the default.")) parser.add_option("--scratch", action="store_true", help=_("Create a scratch image")) @@ -5777,7 +5887,8 @@ def handle_image_build(options, session, args): help=_("Do not attempt to tag package")) parser.add_option("--can-fail", action="store", dest="optional_arches", metavar="ARCH1,ARCH2,...", default="", - help=_("List of archs which are not blocking for build (separated by commas.")) + help=_("List of archs which are not blocking for build " + "(separated by commas.")) parser.add_option("--specfile", metavar="URL", help=_("SCM URL to spec file fragment to use to generate wrapper RPMs")) parser.add_option("--wait", action="store_true", @@ -5826,14 +5937,14 @@ def handle_image_build(options, session, args): else: if len(args) < 5: - parser.error(_("At least five arguments are required: a name, " + - "a version, a build target, a URL to an " + + parser.error(_("At least five arguments are required: a name, " + "a version, a build target, a URL to an " "install tree, and 1 or more architectures.")) if not task_options.ksurl and not task_options.kickstart: parser.error(_('You must specify --kickstart')) if not task_options.distro: parser.error( - _("You must specify --distro. Examples: Fedora-16, RHEL-6.4, " + + _("You must specify --distro. Examples: Fedora-16, RHEL-6.4, " "SL-6.4 or CentOS-6.4")) return _build_image_oz(options, task_options, session, args) @@ -5994,17 +6105,17 @@ def handle_win_build(options, session, args): usage = _("usage: %prog win-build [options] ") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--winspec", metavar="URL", - help=_("SCM URL to retrieve the build descriptor from. " + - "If not specified, the winspec must be in the root directory " + + help=_("SCM URL to retrieve the build descriptor from. " + "If not specified, the winspec must be in the root directory " "of the source repository.")) parser.add_option("--patches", metavar="URL", - help=_("SCM URL of a directory containing patches to apply " + + help=_("SCM URL of a directory containing patches to apply " "to the sources before building")) parser.add_option("--cpus", type="int", - help=_("Number of cpus to allocate to the build VM " + + help=_("Number of cpus to allocate to the build VM " "(requires admin access)")) parser.add_option("--mem", type="int", - help=_("Amount of memory (in megabytes) to allocate to the build VM " + + help=_("Amount of memory (in megabytes) to allocate to the build VM " "(requires admin access)")) parser.add_option("--static-mac", action="store_true", help=_("Retain the original MAC address when cloning the VM")) @@ -6025,7 +6136,8 @@ def handle_win_build(options, session, args): help=_("Do not print the task information"), default=options.quiet) (build_opts, args) = parser.parse_args(args) if len(args) != 3: - parser.error(_("Exactly three arguments (a build target, a SCM URL, and a VM name) are required")) + parser.error( + _("Exactly three arguments (a build target, a SCM URL, and a VM name) are required")) activate_session(session, options) target = args[0] if target.lower() == "none" and build_opts.repo_id: @@ -6121,10 +6233,12 @@ def handle_cancel(goptions, session, args): def handle_set_task_priority(goptions, session, args): "[admin] Set task priority" - usage = _("usage: %prog set-task-priority [options] --priority= [ ...]") + usage = _("usage: %prog set-task-priority [options] --priority= " + "[ ...]") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--priority", type="int", help=_("New priority")) - parser.add_option("--recurse", action="store_true", default=False, help=_("Change priority of child tasks as well")) + parser.add_option("--recurse", action="store_true", default=False, + help=_("Change priority of child tasks as well")) (options, args) = parser.parse_args(args) if len(args) == 0: parser.error(_("You must specify at least one task id")) @@ -6213,7 +6327,8 @@ def handle_set_pkg_owner_global(goptions, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--verbose", action='store_true', help=_("List changes")) parser.add_option("--test", action='store_true', help=_("Test mode")) - parser.add_option("--old-user", "--from", action="store", help=_("Only change ownership for packages belonging to this user")) + parser.add_option("--old-user", "--from", action="store", + help=_("Only change ownership for packages belonging to this user")) (options, args) = parser.parse_args(args) if options.old_user: if len(args) < 1: @@ -6256,11 +6371,13 @@ def handle_set_pkg_owner_global(goptions, session, args): else: if options.test: print("Would have changed owner for %s in tag %s: %s -> %s" - % (entry['package_name'], entry['tag_name'], entry['owner_name'], user['name'])) + % (entry['package_name'], entry['tag_name'], entry['owner_name'], + user['name'])) continue if options.verbose: print("Changing owner for %s in tag %s: %s -> %s" - % (entry['package_name'], entry['tag_name'], entry['owner_name'], user['name'])) + % (entry['package_name'], entry['tag_name'], entry['owner_name'], + user['name'])) session.packageListSetOwner(entry['tag_id'], entry['package_name'], user['id']) @@ -6311,8 +6428,9 @@ def anon_handle_watch_logs(goptions, session, args): usage = _("usage: %prog watch-logs [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--log", help=_("Watch only a specific log")) - parser.add_option("--mine", action="store_true", help=_("Watch logs for " - "all your tasks, task_id arguments are forbidden in this case.")) + parser.add_option("--mine", action="store_true", + help=_("Watch logs for all your tasks, task_id arguments are forbidden in " + "this case.")) parser.add_option("--follow", action="store_true", help=_("Follow spawned child tasks")) (options, args) = parser.parse_args(args) activate_session(session, goptions) @@ -6376,7 +6494,9 @@ def handle_tag_build(opts, session, args): parser.add_option("--nowait", action="store_true", help=_("Do not wait on task")) (options, args) = parser.parse_args(args) if len(args) < 2: - parser.error(_("This command takes at least two arguments: a tag name/ID and one or more package n-v-r's")) + parser.error( + _("This command takes at least two arguments: a tag name/ID and one or more package " + "n-v-r's")) activate_session(session, opts) tasks = [] for pkg in args[1:]: @@ -6398,13 +6518,18 @@ def handle_move_build(opts, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--force", action="store_true", help=_("force operation")) parser.add_option("--nowait", action="store_true", help=_("do not wait on tasks")) - parser.add_option("--all", action="store_true", help=_("move all instances of a package, 's are package names")) + parser.add_option("--all", action="store_true", + help=_("move all instances of a package, 's are package names")) (options, args) = parser.parse_args(args) if len(args) < 3: if options.all: - parser.error(_("This command, with --all, takes at least three arguments: two tags and one or more package names")) + parser.error( + _("This command, with --all, takes at least three arguments: two tags and one or " + "more package names")) else: - parser.error(_("This command takes at least three arguments: two tags and one or more package n-v-r's")) + parser.error( + _("This command takes at least three arguments: two tags and one or more package " + "n-v-r's")) activate_session(session, opts) tasks = [] builds = [] @@ -6442,8 +6567,10 @@ def handle_untag_build(goptions, session, args): "[bind] Remove a tag from one or more builds" usage = _("usage: %prog untag-build [options] [ ...]") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--all", action="store_true", help=_("untag all versions of the package in this tag")) - parser.add_option("--non-latest", action="store_true", help=_("untag all versions of the package in this tag except the latest")) + parser.add_option("--all", action="store_true", + help=_("untag all versions of the package in this tag")) + parser.add_option("--non-latest", action="store_true", + help=_("untag all versions of the package in this tag except the latest")) parser.add_option("-n", "--test", action="store_true", help=_("test mode")) parser.add_option("-v", "--verbose", action="store_true", help=_("print details")) parser.add_option("--force", action="store_true", help=_("force operation")) @@ -6452,7 +6579,9 @@ def handle_untag_build(goptions, session, args): if len(args) < 1: parser.error(_("Please specify a tag")) elif len(args) < 2: - parser.error(_("This command takes at least two arguments: a tag name/ID and one or more package n-v-r's")) + parser.error( + _("This command takes at least two arguments: a tag name/ID and one or more package " + "n-v-r's")) activate_session(session, goptions) tag = session.getTag(args[0]) if not tag: @@ -6526,8 +6655,11 @@ def anon_handle_download_build(options, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--arch", "-a", dest="arches", metavar="ARCH", action="append", default=[], help=_("Only download packages for this arch (may be used multiple times)")) - parser.add_option("--type", help=_("Download archives of the given type, rather than rpms (maven, win, or image)")) - parser.add_option("--latestfrom", dest="latestfrom", help=_("Download the latest build from this tag")) + parser.add_option("--type", + help=_("Download archives of the given type, rather than rpms " + "(maven, win, or image)")) + parser.add_option("--latestfrom", dest="latestfrom", + help=_("Download the latest build from this tag")) parser.add_option("--debuginfo", action="store_true", help=_("Also download -debuginfo rpms")) parser.add_option("--task-id", action="store_true", help=_("Interperet id as a task id")) parser.add_option("--rpm", action="store_true", help=_("Download the given rpm")) @@ -6562,7 +6694,8 @@ def anon_handle_download_build(options, session, args): if suboptions.latestfrom: # We want the latest build, not a specific build try: - builds = session.listTagged(suboptions.latestfrom, latest=True, package=build, type=suboptions.type) + builds = session.listTagged(suboptions.latestfrom, latest=True, package=build, + type=suboptions.type) except koji.GenericError as data: print("Error finding latest build: %s" % data) return 1 @@ -6630,7 +6763,8 @@ def anon_handle_download_build(options, session, args): rpms = session.listRPMs(buildID=info['id'], arches=arches) if not rpms: if arches: - print("No %s packages available for %s" % (" or ".join(arches), koji.buildLabel(info))) + print("No %s packages available for %s" % + (" or ".join(arches), koji.buildLabel(info))) else: print("No packages available for %s" % koji.buildLabel(info)) return 1 @@ -6706,7 +6840,8 @@ def anon_handle_download_logs(options, session, args): offset = 0 try: while contents: - contents = session.downloadTaskOutput(task_id, filename, offset=offset, size=blocksize, volume=volume) + contents = session.downloadTaskOutput(task_id, filename, offset=offset, + size=blocksize, volume=volume) offset += len(contents) if contents: fd.write(contents) @@ -6774,7 +6909,8 @@ def anon_handle_download_task(options, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--arch", dest="arches", metavar="ARCH", action="append", default=[], help=_("Only download packages for this arch (may be used multiple times)")) - parser.add_option("--logs", dest="logs", action="store_true", default=False, help=_("Also download build logs")) + parser.add_option("--logs", dest="logs", action="store_true", default=False, + help=_("Also download build logs")) parser.add_option("--topurl", metavar="URL", default=options.topurl, help=_("URL under which Koji files are accessible")) parser.add_option("--noprogress", action="store_true", @@ -6854,7 +6990,8 @@ def anon_handle_download_task(options, session, args): if '..' in filename: error(_('Invalid file name: %s') % filename) url = '%s/%s/%s' % (pathinfo.work(volume), pathinfo.taskrelpath(task["id"]), filename) - download_file(url, new_filename, suboptions.quiet, suboptions.noprogress, len(downloads), number) + download_file(url, new_filename, suboptions.quiet, suboptions.noprogress, len(downloads), + number) def anon_handle_wait_repo(options, session, args): @@ -6862,10 +6999,16 @@ def anon_handle_wait_repo(options, session, args): usage = _("usage: %prog wait-repo [options] ") parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--build", metavar="NVR", dest="builds", action="append", default=[], - help=_("Check that the given build is in the newly-generated repo (may be used multiple times)")) - parser.add_option("--target", action="store_true", help=_("Interpret the argument as a build target name")) - parser.add_option("--timeout", type="int", help=_("Amount of time to wait (in minutes) before giving up (default: 120)"), default=120) - parser.add_option("--quiet", action="store_true", help=_("Suppress output, success or failure will be indicated by the return value only"), default=options.quiet) + help=_("Check that the given build is in the newly-generated repo " + "(may be used multiple times)")) + parser.add_option("--target", action="store_true", + help=_("Interpret the argument as a build target name")) + parser.add_option("--timeout", type="int", default=120, + help=_("Amount of time to wait (in minutes) before giving up " + "(default: 120)")) + parser.add_option("--quiet", action="store_true", default=options.quiet, + help=_("Suppress output, success or failure will be indicated by the return " + "value only")) (suboptions, args) = parser.parse_args(args) start = time.time() @@ -6906,24 +7049,30 @@ def anon_handle_wait_repo(options, session, args): else: present_nvr = [x["nvr"] for x in data][0] if present_nvr != "%s-%s-%s" % (nvr["name"], nvr["version"], nvr["release"]): - print("Warning: nvr %s-%s-%s is not current in tag %s\n latest build in %s is %s" % (nvr["name"], nvr["version"], nvr["release"], tag, tag, present_nvr)) + print( + "Warning: nvr %s-%s-%s is not current in tag %s\n latest build in %s is %s" % + (nvr["name"], nvr["version"], nvr["release"], tag, tag, present_nvr)) last_repo = None repo = session.getRepo(tag_id) while True: if builds and repo and repo != last_repo: - if koji.util.checkForBuilds(session, tag_id, builds, repo['create_event'], latest=True): + if koji.util.checkForBuilds(session, tag_id, builds, repo['create_event'], + latest=True): if not suboptions.quiet: - print("Successfully waited %s for %s to appear in the %s repo" % (koji.util.duration(start), koji.util.printList(suboptions.builds), tag)) + print("Successfully waited %s for %s to appear in the %s repo" % + (koji.util.duration(start), koji.util.printList(suboptions.builds), tag)) return if (time.time() - start) >= (suboptions.timeout * 60.0): if not suboptions.quiet: if builds: - print("Unsuccessfully waited %s for %s to appear in the %s repo" % (koji.util.duration(start), koji.util.printList(suboptions.builds), tag)) + print("Unsuccessfully waited %s for %s to appear in the %s repo" % + (koji.util.duration(start), koji.util.printList(suboptions.builds), tag)) else: - print("Unsuccessfully waited %s for a new %s repo" % (koji.util.duration(start), tag)) + print("Unsuccessfully waited %s for a new %s repo" % + (koji.util.duration(start), tag)) return 1 time.sleep(options.poll_interval) @@ -6933,7 +7082,8 @@ def anon_handle_wait_repo(options, session, args): if not builds: if repo != last_repo: if not suboptions.quiet: - print("Successfully waited %s for a new %s repo" % (koji.util.duration(start), tag)) + print("Successfully waited %s for a new %s repo" % + (koji.util.duration(start), tag)) return @@ -6941,11 +7091,14 @@ def handle_regen_repo(options, session, args): "[admin] Force a repo to be regenerated" usage = _("usage: %prog regen-repo [options] ") parser = OptionParser(usage=get_usage_str(usage)) - parser.add_option("--target", action="store_true", help=_("Interpret the argument as a build target name")) + parser.add_option("--target", action="store_true", + help=_("Interpret the argument as a build target name")) parser.add_option("--nowait", action="store_true", help=_("Don't wait on for regen to finish")) parser.add_option("--debuginfo", action="store_true", help=_("Include debuginfo rpms in repo")) - parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in each of repos")) - parser.add_option("--separate-source", "--separate-src", action="store_true", help=_("Include source rpms in separate src repo")) + parser.add_option("--source", "--src", action="store_true", + help=_("Include source rpms in each of repos")) + parser.add_option("--separate-source", "--separate-src", action="store_true", + help=_("Include source rpms in separate src repo")) (suboptions, args) = parser.parse_args(args) if len(args) == 0: parser.error(_("A tag name must be specified")) @@ -7006,8 +7159,8 @@ def handle_dist_repo(options, session, args): help=_('For RPMs not signed with a desired key, fall back to the ' 'primary copy')) parser.add_option("-a", "--arch", action='append', default=[], - help=_("Indicate an architecture to consider. The default is all " + - "architectures associated with the given tag. This option may " + + help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " "be specified multiple times.")) parser.add_option("--with-src", action='store_true', help='Also generate a src repo') parser.add_option("--split-debuginfo", action='store_true', default=False, @@ -7035,7 +7188,8 @@ def handle_dist_repo(options, session, args): parser.add_option('--zck', action='store_true', default=False, help=_('Generate zchunk files as well as the standard repodata')) parser.add_option('--zck-dict-dir', action='store', default=None, - help=_('Directory containing compression dictionaries for use by zchunk (on builder)')) + help=_('Directory containing compression dictionaries for use by zchunk ' + '(on builder)')) task_opts, args = parser.parse_args(args) if len(args) < 1: parser.error(_('You must provide a tag to generate the repo from')) @@ -7210,7 +7364,11 @@ def anon_handle_list_notifications(goptions, session, args): user_id = None mask = "%(id)6s %(tag)-25s %(package)-25s %(email)-20s %(success)-12s" - headers = {'id': 'ID', 'tag': 'Tag', 'package': 'Package', 'email': 'E-mail', 'success': 'Success-only'} + headers = {'id': 'ID', + 'tag': 'Tag', + 'package': 'Package', + 'email': 'E-mail', + 'success': 'Success-only'} head = mask % headers notifications = session.getBuildNotifications(user_id) if notifications: @@ -7300,7 +7458,8 @@ def handle_add_notification(goptions, session, args): def handle_remove_notification(goptions, session, args): "[monitor] Remove user's notifications" - usage = _("usage: %prog remove-notification [options] [ ...]") + usage = _("usage: %prog remove-notification [options] " + "[ ...]") parser = OptionParser(usage=get_usage_str(usage)) (options, args) = parser.parse_args(args) @@ -7422,7 +7581,7 @@ def handle_block_notification(goptions, session, args): tag_id = None for block in session.getBuildNotificationBlocks(user_id): - if (block['package_id'] == package_id and block['tag_id'] == tag_id): + if block['package_id'] == package_id and block['tag_id'] == tag_id: parser.error('Notification already exists.') session.createNotificationBlock(user_id, package_id, tag_id) @@ -7430,7 +7589,8 @@ def handle_block_notification(goptions, session, args): def handle_unblock_notification(goptions, session, args): "[monitor] Unblock user's notification" - usage = _("usage: %prog unblock-notification [options] [ ...]") + usage = _("usage: %prog unblock-notification [options] " + "[ ...]") parser = OptionParser(usage=get_usage_str(usage)) (options, args) = parser.parse_args(args) diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index dc4b663..c2cc6c8 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -112,7 +112,8 @@ def ensure_connection(session): except requests.exceptions.ConnectionError: error(_("Error: Unable to connect to server")) if ret != koji.API_VERSION: - warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) + warn(_("WARNING: The server is at API version %d and " + "the client is at %d" % (ret, koji.API_VERSION))) def print_task_headers(): @@ -194,7 +195,8 @@ class TaskWatcher(object): laststate = last['state'] if laststate != state: if not self.quiet: - print("%s: %s -> %s" % (self.str(), self.display_state(last), self.display_state(self.info))) + print("%s: %s -> %s" % (self.str(), self.display_state(last), + self.display_state(self.info))) return True return False else: @@ -277,9 +279,9 @@ def watch_tasks(session, tasklist, quiet=False, poll_interval=60, ki_handler=Non tlist = ['%s: %s' % (t.str(), t.display_state(t.info)) for t in tasks.values() if not t.is_done()] print( - """Tasks still running. You can continue to watch with the '%s watch-task' command. -Running Tasks: -%s""" % (progname, '\n'.join(tlist))) + "Tasks still running. You can continue to watch with the" + " '%s watch-task' command.\n" + "Running Tasks:\n%s" % (progname, '\n'.join(tlist))) sys.stdout.flush() rv = 0 try: @@ -302,7 +304,8 @@ Running Tasks: for child in session.getTaskChildren(task_id): child_id = child['id'] if child_id not in tasks.keys(): - tasks[child_id] = TaskWatcher(child_id, session, task.level + 1, quiet=quiet) + tasks[child_id] = TaskWatcher(child_id, session, task.level + 1, + quiet=quiet) tasks[child_id].update() # If we found new children, go through the list again, # in case they have children also @@ -370,7 +373,8 @@ def watch_logs(session, tasklist, opts, poll_interval): if (log, volume) not in taskoffsets: taskoffsets[(log, volume)] = 0 - contents = session.downloadTaskOutput(task_id, log, taskoffsets[(log, volume)], 16384, volume=volume) + contents = session.downloadTaskOutput(task_id, log, taskoffsets[(log, volume)], + 16384, volume=volume) taskoffsets[(log, volume)] += len(contents) if contents: currlog = "%d:%s:%s:" % (task_id, volume, log) @@ -452,7 +456,9 @@ def _progress_callback(uploaded, total, piece, time, total_time): speed = _format_size(float(total) / float(total_time)) + "/sec" # write formated string and flush - sys.stdout.write("[% -36s] % 4s % 8s % 10s % 14s\r" % ('=' * (int(percent_done * 36)), percent_done_str, elapsed, data_done, speed)) + sys.stdout.write("[% -36s] % 4s % 8s % 10s % 14s\r" % ('=' * (int(percent_done * 36)), + percent_done_str, elapsed, data_done, + speed)) sys.stdout.flush() @@ -520,7 +526,8 @@ def _download_progress(download_t, download_d): percent_done_str = "%3d%%" % (percent_done * 100) data_done = _format_size(download_d) - sys.stdout.write("[% -36s] % 4s % 10s\r" % ('=' * (int(percent_done * 36)), percent_done_str, data_done)) + sys.stdout.write("[% -36s] % 4s % 10s\r" % ('=' * (int(percent_done * 36)), percent_done_str, + data_done)) sys.stdout.flush() @@ -560,13 +567,16 @@ def activate_session(session, options): elif options.authtype == "ssl" or os.path.isfile(options.cert) and options.authtype is None: # authenticate using SSL client cert session.ssl_login(options.cert, None, options.serverca, proxyuser=runas) - elif options.authtype == "password" or getattr(options, 'user', None) and options.authtype is None: + elif options.authtype == "password" \ + or getattr(options, 'user', None) \ + and options.authtype is None: # authenticate using user/password session.login() elif options.authtype == "kerberos" or has_krb_creds() and options.authtype is None: try: if getattr(options, 'keytab', None) and getattr(options, 'principal', None): - session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=runas) + session.krb_login(principal=options.principal, keytab=options.keytab, + proxyuser=runas) else: session.krb_login(proxyuser=runas) except socket.error as e: diff --git a/hub/kojihub.py b/hub/kojihub.py index 4fc9d35..b350cca 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -636,10 +636,12 @@ def make_task(method, arglist, **opts): opts['request'] = koji.xmlrpcplus.dumps(tuple(arglist), methodname=method) opts['state'] = koji.TASK_STATES['FREE'] opts['method'] = method - koji.plugin.run_callbacks('preTaskStateChange', attribute='state', old=None, new='FREE', info=opts) + koji.plugin.run_callbacks( + 'preTaskStateChange', attribute='state', old=None, new='FREE', info=opts) # stick it in the database - idata = dslice(opts, ['state', 'owner', 'method', 'request', 'priority', 'parent', 'label', 'channel_id', 'arch']) + idata = dslice(opts, ['state', 'owner', 'method', 'request', 'priority', 'parent', 'label', + 'channel_id', 'arch']) if opts.get('assign'): idata['state'] = koji.TASK_STATES['ASSIGNED'] idata['host_id'] = opts['assign'] @@ -647,7 +649,8 @@ def make_task(method, arglist, **opts): insert.execute() task_id = _singleValue("SELECT currval('task_id_seq')", strict=True) opts['id'] = task_id - koji.plugin.run_callbacks('postTaskStateChange', attribute='state', old=None, new='FREE', info=opts) + koji.plugin.run_callbacks( + 'postTaskStateChange', attribute='state', old=None, new='FREE', info=opts) return task_id @@ -660,8 +663,8 @@ def eventCondition(event, table=None): if event is None: return """(%(table)sactive = TRUE)""" % locals() elif isinstance(event, six.integer_types): - return """(%(table)screate_event <= %(event)d AND ( %(table)srevoke_event IS NULL OR %(event)d < %(table)srevoke_event ))""" \ - % locals() + return "(%(table)screate_event <= %(event)d AND ( %(table)srevoke_event IS NULL OR " \ + "%(event)d < %(table)srevoke_event ))" % locals() else: raise koji.GenericError("Invalid event: %r" % event) @@ -681,7 +684,8 @@ def readGlobalInheritance(event=None): def readInheritanceData(tag_id, event=None): c = context.cnx.cursor() - fields = ('parent_id', 'name', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter') + fields = ('parent_id', 'name', 'priority', 'maxdepth', 'intransitive', 'noconfig', + 'pkg_filter') q = """SELECT %s FROM tag_inheritance JOIN tag ON parent_id = id WHERE %s AND tag_id = %%(tag_id)i ORDER BY priority @@ -697,7 +701,8 @@ def readInheritanceData(tag_id, event=None): def readDescendantsData(tag_id, event=None): c = context.cnx.cursor() - fields = ('tag_id', 'parent_id', 'name', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter') + fields = ('tag_id', 'parent_id', 'name', 'priority', 'maxdepth', 'intransitive', 'noconfig', + 'pkg_filter') q = """SELECT %s FROM tag_inheritance JOIN tag ON tag_id = id WHERE %s AND parent_id = %%(tag_id)i ORDER BY priority @@ -778,7 +783,8 @@ def _writeInheritanceData(tag_id, changes, clear=False): continue # oops, duplicate entries for a single priority dup_ids = [link['parent_id'] for link in dups] - raise koji.GenericError("Inheritance priorities must be unique (pri %s: %r )" % (pri, dup_ids)) + raise koji.GenericError("Inheritance priorities must be unique (pri %s: %r )" % + (pri, dup_ids)) for parent_id, link in six.iteritems(data): if not link.get('is_update'): continue @@ -809,11 +815,13 @@ def readFullInheritance(tag_id, event=None, reverse=False, stops=None, jumps=Non if jumps is None: jumps = {} order = [] - readFullInheritanceRecurse(tag_id, event, order, stops, {}, {}, 0, None, False, [], reverse, jumps) + readFullInheritanceRecurse(tag_id, event, order, stops, {}, {}, 0, None, False, [], reverse, + jumps) return order -def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdepth, maxdepth, noconfig, pfilter, reverse, jumps): +def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdepth, maxdepth, + noconfig, pfilter, reverse, jumps): if maxdepth is not None and maxdepth < 1: return # note: maxdepth is relative to where we are, but currdepth is absolute from @@ -905,7 +913,8 @@ def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdept if link['intransitive'] and reverse: # add link, but don't follow it continue - readFullInheritanceRecurse(id, event, order, prunes, top, hist, currdepth, nextdepth, noconfig, filter, reverse, jumps) + readFullInheritanceRecurse(id, event, order, prunes, top, hist, currdepth, nextdepth, + noconfig, filter, reverse, jumps) # tag-package operations # add @@ -953,7 +962,8 @@ def _pkglist_add(tag_id, pkg_id, owner, block, extra_arches): _pkglist_owner_add(tag_id, pkg_id, owner) -def pkglist_add(taginfo, pkginfo, owner=None, block=None, extra_arches=None, force=False, update=False): +def pkglist_add(taginfo, pkginfo, owner=None, block=None, extra_arches=None, force=False, + update=False): """Add to (or update) package list for tag""" return _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, update, policy=True) @@ -1064,9 +1074,11 @@ def _direct_pkglist_remove(taginfo, pkginfo, force=False, policy=False): if not (force and context.session.hasPerm('admin')): assert_policy('package_list', policy_data) user = get_user(context.session.user_id) - koji.plugin.run_callbacks('prePackageListChange', action='remove', tag=tag, package=pkg, user=user) + koji.plugin.run_callbacks( + 'prePackageListChange', action='remove', tag=tag, package=pkg, user=user) _pkglist_remove(tag['id'], pkg['id']) - koji.plugin.run_callbacks('postPackageListChange', action='remove', tag=tag, package=pkg, user=user) + koji.plugin.run_callbacks( + 'postPackageListChange', action='remove', tag=tag, package=pkg, user=user) def pkglist_block(taginfo, pkginfo, force=False): @@ -1094,7 +1106,8 @@ def pkglist_unblock(taginfo, pkginfo, force=False): if not (force and context.session.hasPerm('admin')): assert_policy('package_list', policy_data) user = get_user(context.session.user_id) - koji.plugin.run_callbacks('prePackageListChange', action='unblock', tag=tag, package=pkg, user=user) + koji.plugin.run_callbacks( + 'prePackageListChange', action='unblock', tag=tag, package=pkg, user=user) tag_id = tag['id'] pkg_id = pkg['id'] pkglist = readPackageList(tag_id, pkgID=pkg_id, inherit=True) @@ -1114,7 +1127,8 @@ def pkglist_unblock(taginfo, pkginfo, force=False): pkglist = readPackageList(tag_id, pkgID=pkg_id, inherit=True) if pkg_id not in pkglist or pkglist[pkg_id]['blocked']: _pkglist_add(tag_id, pkg_id, previous['owner_id'], False, previous['extra_arches']) - koji.plugin.run_callbacks('postPackageListChange', action='unblock', tag=tag, package=pkg, user=user) + koji.plugin.run_callbacks( + 'postPackageListChange', action='unblock', tag=tag, package=pkg, user=user) def pkglist_setowner(taginfo, pkginfo, owner, force=False): @@ -1127,7 +1141,8 @@ def pkglist_setarches(taginfo, pkginfo, arches, force=False): pkglist_add(taginfo, pkginfo, extra_arches=arches, force=force, update=True) -def readPackageList(tagID=None, userID=None, pkgID=None, event=None, inherit=False, with_dups=False): +def readPackageList(tagID=None, userID=None, pkgID=None, event=None, inherit=False, + with_dups=False): """Returns the package list for the specified tag or user. One of (tagID,userID,pkgID) must be specified @@ -1268,7 +1283,8 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None): packageinfo = lookup_package(package) if not packageinfo: raise koji.GenericError('invalid package: %s' % package) - fields.extend(['users.id', 'users.name', 'tag_packages.blocked', 'tag_packages.extra_arches']) + fields.extend( + ['users.id', 'users.name', 'tag_packages.blocked', 'tag_packages.extra_arches']) aliases.extend(['owner_id', 'owner_name', 'blocked', 'extra_arches']) joins.append('tag_packages ON tag.id = tag_packages.tag_id') clauses.append('tag_packages.active = true') @@ -1286,7 +1302,8 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None): return query.iterate() -def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, type=None): +def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, + type=None): """Returns a list of builds for specified tag set inherit=True to follow inheritance @@ -1314,7 +1331,8 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, # these values are used for each iteration fields = [('tag.id', 'tag_id'), ('tag.name', 'tag_name'), ('build.id', 'id'), ('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), - ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), @@ -1358,7 +1376,8 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, JOIN volume ON volume.id = build.volume_id WHERE %s AND tag_id=%%(tagid)s AND build.state=%%(st_complete)i - """ % (', '.join([pair[0] for pair in fields]), type_join, eventCondition(event, 'tag_listing')) + """ % (', '.join([pair[0] for pair in fields]), type_join, + eventCondition(event, 'tag_listing')) if package: q += """AND package.name = %(package)s """ @@ -1393,7 +1412,8 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, return builds -def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, latest=True, rpmsigs=False, owner=None, type=None): +def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, latest=True, + rpmsigs=False, owner=None, type=None): """Returns a list of rpms for specified tag set inherit=True to follow inheritance @@ -1410,7 +1430,8 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late # (however, it is fairly quick) taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] - builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) + builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, + owner=owner, type=type) # index builds build_idx = dict([(b['build_id'], b) for b in builds]) @@ -1505,7 +1526,8 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] # If type == 'maven', we require that both the build *and* the archive have Maven metadata - builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, type=type) + builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, + type=type) # index builds build_idx = dict([(b['build_id'], b) for b in builds]) @@ -1699,7 +1721,8 @@ def _untag_build(tag, build, user_id=None, strict=True, force=False): def _direct_untag_build(tag, build, user, strict=True, force=False): """Directly untag a build. No access check or value lookup.""" - koji.plugin.run_callbacks('preUntag', tag=tag, build=build, user=user, force=force, strict=strict) + koji.plugin.run_callbacks( + 'preUntag', tag=tag, build=build, user=user, force=force, strict=strict) values = {'tag_id': tag['id'], 'build_id': build['id']} update = UpdateProcessor('tag_listing', values=values, clauses=['tag_id=%(tag_id)i', 'build_id=%(build_id)i']) @@ -1708,7 +1731,8 @@ def _direct_untag_build(tag, build, user, strict=True, force=False): if count == 0 and strict: nvr = "%(name)s-%(version)s-%(release)s" % build raise koji.TagError("build %s not in tag %s" % (nvr, tag['name'])) - koji.plugin.run_callbacks('postUntag', tag=tag, build=build, user=user, force=force, strict=strict) + koji.plugin.run_callbacks( + 'postUntag', tag=tag, build=build, user=user, force=force, strict=strict) # tag-group operations @@ -1909,7 +1933,9 @@ def _grp_pkg_add(taginfo, grpinfo, pkg_name, block, force, **opts): opts['blocked'] = block # revoke old entry (if present) update = UpdateProcessor('group_package_listing', values=opts, - clauses=['group_id=%(group_id)s', 'tag_id=%(tag_id)s', 'package=%(package)s']) + clauses=['group_id=%(group_id)s', + 'tag_id=%(tag_id)s', + 'package=%(package)s']) update.make_revoke() update.execute() # add new entry @@ -1934,7 +1960,9 @@ def _grp_pkg_remove(taginfo, grpinfo, pkg_name, force): tag_id = get_tag_id(taginfo, strict=True) grp_id = get_group_id(grpinfo, strict=True) update = UpdateProcessor('group_package_listing', values=locals(), - clauses=['package=%(pkg_name)s', 'tag_id=%(tag_id)s', 'group_id = %(grp_id)s']) + clauses=['package=%(pkg_name)s', + 'tag_id=%(tag_id)s', + 'group_id = %(grp_id)s']) update.make_revoke() update.execute() @@ -2033,7 +2061,9 @@ def _grp_req_add(taginfo, grpinfo, reqinfo, block, force, **opts): opts['blocked'] = block # revoke old entry (if present) update = UpdateProcessor('group_req_listing', values=opts, - clauses=['group_id=%(group_id)s', 'tag_id=%(tag_id)s', 'req_id=%(req_id)s']) + clauses=['group_id=%(group_id)s', + 'tag_id=%(tag_id)s', + 'req_id=%(req_id)s']) update.make_revoke() update.execute() # add new entry @@ -2059,7 +2089,9 @@ def _grp_req_remove(taginfo, grpinfo, reqinfo, force): grp_id = get_group_id(grpinfo, strict=True) req_id = get_group_id(reqinfo, strict=True) update = UpdateProcessor('group_req_listing', values=locals(), - clauses=['req_id=%(req_id)s', 'tag_id=%(tag_id)s', 'group_id = %(grp_id)s']) + clauses=['req_id=%(req_id)s', + 'tag_id=%(tag_id)s', + 'group_id = %(grp_id)s']) update.make_revoke() update.execute() @@ -2184,7 +2216,8 @@ def get_tag_groups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True return groups -def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, incl_blocked=False): +def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, + incl_blocked=False): """Return group data for the tag with blocked entries removed Also scrubs data into an xmlrpc-safe format (no integer keys) @@ -2199,12 +2232,14 @@ def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, if incl_blocked: group['packagelist'] = to_list(group['packagelist'].values()) else: - group['packagelist'] = [x for x in group['packagelist'].values() if not x['blocked']] + group['packagelist'] = [x for x in group['packagelist'].values() + if not x['blocked']] if 'grouplist' in group: if incl_blocked: group['grouplist'] = to_list(group['grouplist'].values()) else: - group['grouplist'] = [x for x in group['grouplist'].values() if not x['blocked']] + group['grouplist'] = [x for x in group['grouplist'].values() + if not x['blocked']] # filter blocked entries and collapse to a list if incl_blocked: return groups @@ -2245,7 +2280,8 @@ def add_host_to_channel(hostname, channel_name, create=False): channels = list_channels(host_id) for channel in channels: if channel['id'] == channel_id: - raise koji.GenericError('host %s is already subscribed to the %s channel' % (hostname, channel_name)) + raise koji.GenericError('host %s is already subscribed to the %s channel' % + (hostname, channel_name)) insert = InsertProcessor('host_channels') insert.set(host_id=host_id, channel_id=channel_id) insert.make_create() @@ -2268,7 +2304,8 @@ def remove_host_from_channel(hostname, channel_name): found = True break if not found: - raise koji.GenericError('host %s is not subscribed to the %s channel' % (hostname, channel_name)) + raise koji.GenericError('host %s is not subscribed to the %s channel' % + (hostname, channel_name)) values = {'host_id': host_id, 'channel_id': channel_id} clauses = ['host_id = %(host_id)i AND channel_id = %(channel_id)i'] @@ -2501,8 +2538,9 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa logger = logging.getLogger("koji.hub.repo_init") state = koji.REPO_INIT tinfo = get_tag(tag, strict=True, event=event) - koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo, - event=event, repo_id=None, with_separate_src=with_separate_src) + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=with_src, + with_debuginfo=with_debuginfo, event=event, repo_id=None, + with_separate_src=with_separate_src) tag_id = tinfo['id'] repo_arches = {} if with_separate_src: @@ -2639,8 +2677,9 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa for artifact_dir, artifacts in six.iteritems(artifact_dirs): _write_maven_repo_metadata(artifact_dir, artifacts) - koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo, - event=event, repo_id=repo_id, with_separate_src=with_separate_src) + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=with_src, + with_debuginfo=with_debuginfo, event=event, repo_id=repo_id, + with_separate_src=with_separate_src) return [repo_id, event_id] @@ -3188,13 +3227,15 @@ def lookup_build_target(info, strict=False, create=False): return lookup_name('build_target', info, strict, create) -def create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_support=False, maven_include_all=False, extra=None): +def create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_support=False, + maven_include_all=False, extra=None): """Create a new tag""" context.session.assertPerm('tag') return _create_tag(name, parent, arches, perm, locked, maven_support, maven_include_all, extra) -def _create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_support=False, maven_include_all=False, extra=None): +def _create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_support=False, + maven_include_all=False, extra=None): """Create a new tag, without access check""" max_name_length = 256 @@ -3416,7 +3457,8 @@ WHERE id = %(tagID)i""" if 'remove_extra' in kwargs: for removed in kwargs['remove_extra']: if removed in kwargs['extra']: - raise koji.GenericError("Can not both add/update and remove tag-extra: '%s'" % removed) + raise koji.GenericError("Can not both add/update and remove tag-extra: '%s'" % + removed) for key in kwargs['extra']: value = kwargs['extra'][key] if key not in tag['extra'] or tag['extra'][key] != value: @@ -3426,7 +3468,8 @@ WHERE id = %(tagID)i""" 'value': json.dumps(kwargs['extra'][key]), } # revoke old entry, if any - update = UpdateProcessor('tag_extra', values=data, clauses=['tag_id = %(tag_id)i', 'key=%(key)s']) + update = UpdateProcessor('tag_extra', values=data, clauses=['tag_id = %(tag_id)i', + 'key=%(key)s']) update.make_revoke() update.execute() # add new entry @@ -3438,14 +3481,16 @@ WHERE id = %(tagID)i""" if 'remove_extra' in kwargs: ne = [e for e in kwargs['remove_extra'] if e not in tag['extra']] if ne: - raise koji.GenericError("Tag: %s doesn't have extra: %s" % (tag['name'], ', '.join(ne))) + raise koji.GenericError("Tag: %s doesn't have extra: %s" % + (tag['name'], ', '.join(ne))) for key in kwargs['remove_extra']: data = { 'tag_id': tag['id'], 'key': key, } # revoke old entry - update = UpdateProcessor('tag_extra', values=data, clauses=['tag_id = %(tag_id)i', 'key=%(key)s']) + update = UpdateProcessor('tag_extra', values=data, clauses=['tag_id = %(tag_id)i', + 'key=%(key)s']) update.make_revoke() update.execute() @@ -3576,7 +3621,8 @@ def edit_external_repo(info, name=None, url=None): existing_id = _singleValue("""SELECT id FROM external_repo WHERE name = %(name)s""", locals(), strict=False) if existing_id is not None: - raise koji.GenericError('name "%s" is already taken by external repo %i' % (name, existing_id)) + raise koji.GenericError('name "%s" is already taken by external repo %i' % + (name, existing_id)) rename = """UPDATE external_repo SET name = %(name)s WHERE id = %(repo_id)i""" _dml(rename, locals()) @@ -3714,7 +3760,8 @@ def get_tag_external_repos(tag_info=None, repo_info=None, event=None): } columns, aliases = zip(*fields.items()) - clauses = [eventCondition(event, table='tag_external_repos'), eventCondition(event, table='external_repo_config')] + clauses = [eventCondition(event, table='tag_external_repos'), + eventCondition(event, table='external_repo_config')] if tag_info: tag = get_tag(tag_info, strict=True, event=event) tag_id = tag['id'] @@ -4035,10 +4082,13 @@ def get_build(buildInfo, strict=False): fields = (('build.id', 'id'), ('build.version', 'version'), ('build.release', 'release'), ('build.id', 'build_id'), - ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), - ('build.task_id', 'task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), - ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), + ('build.task_id', 'task_id'), + ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), + ('package.id', 'package_id'), ('package.name', 'package_name'), + ('package.name', 'name'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), ('EXTRACT(EPOCH FROM events.time)', 'creation_ts'), @@ -4248,7 +4298,8 @@ def get_rpm(rpminfo, strict=False, multi=False): return ret -def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None): +def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, + arches=None, queryOpts=None): """List RPMS. If buildID, imageID and/or buildrootID are specified, restrict the list of RPMs to only those RPMs that are part of that build, or were built in that buildroot. If componentBuildrootID is specified, @@ -4311,7 +4362,8 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID joins.append('archive_rpm_components ON rpminfo.id = archive_rpm_components.rpm_id') if hostID is not None: - joins.append('standard_buildroot ON rpminfo.buildroot_id = standard_buildroot.buildroot_id') + joins.append( + 'standard_buildroot ON rpminfo.buildroot_id = standard_buildroot.buildroot_id') clauses.append('standard_buildroot.host_id = %(hostID)i') if arches is not None: if isinstance(arches, (list, tuple)): @@ -4474,17 +4526,19 @@ def add_btype(name): insert.execute() -def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None, - filename=None, size=None, checksum=None, typeInfo=None, queryOpts=None, imageID=None, - archiveID=None, strict=False): +def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, + type=None, filename=None, size=None, checksum=None, typeInfo=None, + queryOpts=None, imageID=None, archiveID=None, strict=False): """ Retrieve information about archives. If buildID is not null it will restrict the list to archives built by the build with that ID. - If buildrootID is not null it will restrict the list to archives built in the buildroot with that ID. - If componentBuildrootID is not null it will restrict the list to archives that were present in the - buildroot with that ID. + If buildrootID is not null it will restrict the list to archives built in the buildroot with + that ID. + If componentBuildrootID is not null it will restrict the list to archives that were present in + the buildroot with that ID. If hostID is not null it will restrict the list to archives built on the host with that ID. - If filename, size, and/or checksum are not null it will filter the results to entries matching the provided values. + If filename, size, and/or checksum are not null it will filter the results to entries matching + the provided values. Returns a list of maps containing the following keys: @@ -4578,7 +4632,8 @@ def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hos values['imageID'] = imageID joins.append('archive_components ON archiveinfo.id = archive_components.component_id') if hostID is not None: - joins.append('standard_buildroot on archiveinfo.buildroot_id = standard_buildroot.buildroot_id') + joins.append( + 'standard_buildroot on archiveinfo.buildroot_id = standard_buildroot.buildroot_id') clauses.append('standard_buildroot.host_id = %(host_id)i') values['host_id'] = hostID fields.append(['standard_buildroot.host_id', 'host_id']) @@ -5135,7 +5190,10 @@ def edit_host(hostInfo, **kw): update.make_revoke() update.execute() - insert = InsertProcessor('host_config', data=dslice(host, ('arches', 'capacity', 'description', 'comment', 'enabled'))) + insert = InsertProcessor('host_config', + data=dslice(host, + ('arches', 'capacity', 'description', 'comment', + 'enabled'))) insert.set(host_id=host['id']) for change in changes: insert.set(**{change: kw[change]}) @@ -5171,7 +5229,8 @@ def get_channel(channelInfo, strict=False): return _singleRow(query, locals(), fields, strict) -def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID=None, taskID=None, buildrootID=None, queryOpts=None): +def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID=None, taskID=None, + buildrootID=None, queryOpts=None): """Return a list of matching buildroots Optional args: @@ -5204,16 +5263,21 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= ('EXTRACT(EPOCH FROM create_events.time)', 'create_ts'), ('retire_events.id', 'retire_event_id'), ('retire_events.time', 'retire_event_time'), ('EXTRACT(EPOCH FROM retire_events.time)', 'retire_ts'), - ('repo_create.id', 'repo_create_event_id'), ('repo_create.time', 'repo_create_event_time')] + ('repo_create.id', 'repo_create_event_id'), + ('repo_create.time', 'repo_create_event_time')] tables = ['buildroot'] - joins = ['LEFT OUTER JOIN standard_buildroot ON standard_buildroot.buildroot_id = buildroot.id', - 'LEFT OUTER JOIN content_generator ON buildroot.cg_id = content_generator.id', + joins = ['LEFT OUTER JOIN standard_buildroot ' + 'ON standard_buildroot.buildroot_id = buildroot.id', + 'LEFT OUTER JOIN content_generator ' + 'ON buildroot.cg_id = content_generator.id', 'LEFT OUTER JOIN host ON host.id = standard_buildroot.host_id', 'LEFT OUTER JOIN repo ON repo.id = standard_buildroot.repo_id', 'LEFT OUTER JOIN tag ON tag.id = repo.tag_id', - 'LEFT OUTER JOIN events AS create_events ON create_events.id = standard_buildroot.create_event', - 'LEFT OUTER JOIN events AS retire_events ON standard_buildroot.retire_event = retire_events.id', + 'LEFT OUTER JOIN events AS create_events ON ' + 'create_events.id = standard_buildroot.create_event', + 'LEFT OUTER JOIN events AS retire_events ON ' + 'standard_buildroot.retire_event = retire_events.id', 'LEFT OUTER JOIN events AS repo_create ON repo_create.id = repo.create_event'] clauses = [] @@ -5399,7 +5463,8 @@ def _set_build_volume(binfo, volinfo, strict=True): shutil.copytree(olddir, newdir, symlinks=True) # Second, update the db - koji.plugin.run_callbacks('preBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) + koji.plugin.run_callbacks('preBuildStateChange', attribute='volume_id', + old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) update.set(volume_id=volinfo['id']) update.execute() @@ -5422,7 +5487,8 @@ def _set_build_volume(binfo, volinfo, strict=True): relpath = os.path.relpath(newdir, os.path.dirname(basedir)) os.symlink(relpath, basedir) - koji.plugin.run_callbacks('postBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) + koji.plugin.run_callbacks('postBuildStateChange', attribute='volume_id', + old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) def ensure_volume_symlink(binfo): @@ -5573,18 +5639,21 @@ def new_build(data, strict=False): recycle_build(old_binfo, data) # Raises exception if there is a problem return old_binfo['id'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data) + koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, + new=data['state'], info=data) # insert the new data insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', - 'task_id', 'owner', 'start_time', 'completion_time', 'source', 'extra']) + 'task_id', 'owner', 'start_time', 'completion_time', 'source', + 'extra']) if 'cg_id' in data: insert_data['cg_id'] = data['cg_id'] data['id'] = insert_data['id'] = _singleValue("SELECT nextval('build_id_seq')") insert = InsertProcessor('build', data=insert_data) insert.execute() new_binfo = get_build(data['id'], strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=new_binfo) + koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, + new=data['state'], info=new_binfo) # return build_id return data['id'] @@ -5759,10 +5828,13 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) binfo = get_build(build_id, strict=True) st_complete = koji.BUILD_STATES['COMPLETE'] st_old = binfo['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=binfo) + koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, + new=st_complete, info=binfo) for key in ('name', 'version', 'release', 'epoch', 'task_id'): if build[key] != binfo[key]: - raise koji.GenericError("Unable to complete build: %s mismatch (build: %s, rpm: %s)" % (key, binfo[key], build[key])) + raise koji.GenericError( + "Unable to complete build: %s mismatch (build: %s, rpm: %s)" % + (key, binfo[key], build[key])) if binfo['state'] != koji.BUILD_STATES['BUILDING']: raise koji.GenericError("Unable to complete build: state is %s" % koji.BUILD_STATES[binfo['state']]) @@ -5773,7 +5845,8 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) update.set(volume_id=build['volume_id']) update.execute() binfo = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_complete, info=binfo) + koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, + new=st_complete, info=binfo) # now to handle the individual rpms for relpath in [srpm] + rpms: @@ -6143,7 +6216,8 @@ class CG_Importer(object): metadata = self.metadata if metadata['build'].get('build_id'): if len(self.cgs) != 1: - raise koji.GenericError("Reserved builds can handle only single content generator.") + raise koji.GenericError( + "Reserved builds can handle only single content generator.") cg_id = list(self.cgs)[0] build_id = metadata['build']['build_id'] buildinfo = get_build(build_id, strict=True) @@ -6257,7 +6331,8 @@ class CG_Importer(object): source = self.buildinfo.get('source') st_complete = koji.BUILD_STATES['COMPLETE'] st_old = old_info['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=old_info) + koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, + new=st_complete, info=old_info) update = UpdateProcessor('build', clauses=['id=%(build_id)s'], values=self.buildinfo) update.set(state=st_complete, extra=extra, owner=owner, source=source) if self.buildinfo.get('volume_id'): @@ -6267,7 +6342,8 @@ class CG_Importer(object): update.execute() buildinfo = get_build(build_id, strict=True) clear_reservation(build_id) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_complete, info=buildinfo) + koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, + new=st_complete, info=buildinfo) return buildinfo @@ -6396,14 +6472,16 @@ class CG_Importer(object): if archive['checksum'] == comp['checksum']: return archive # else - logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) + logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", + comp) if type_mismatches: logger.error("Match failed with %i type mismatches", type_mismatches) # TODO: allow external archives # XXX - this is a temporary workaround until we can better track external refs logger.warning("IGNORING unmatched archive: %r", comp) return None - # raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + # raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % + # comp) def match_kojifile(self, comp): """Look up the file by archive id and sanity check the other data""" @@ -6437,13 +6515,15 @@ class CG_Importer(object): if fileinfo.get('metadata_only', False): self.metadata_only = True workdir = koji.pathinfo.work() - path = joinpath(workdir, self.directory, fileinfo.get('relpath', ''), fileinfo['filename']) + path = joinpath(workdir, self.directory, fileinfo.get('relpath', ''), + fileinfo['filename']) fileinfo['hub.path'] = path filesize = os.path.getsize(path) if filesize != fileinfo['filesize']: - raise koji.GenericError("File size %s for %s (expected %s) doesn't match. Corrupted upload?" % - (filesize, fileinfo['filename'], fileinfo['filesize'])) + raise koji.GenericError( + "File size %s for %s (expected %s) doesn't match. Corrupted upload?" % + (filesize, fileinfo['filename'], fileinfo['filesize'])) # checksum if fileinfo['checksum_type'] != 'md5': @@ -6459,11 +6539,13 @@ class CG_Importer(object): m.update(contents) if fileinfo['checksum'] != m.hexdigest(): raise koji.GenericError("File checksum mismatch for %s: %s != %s" % - (fileinfo['filename'], fileinfo['checksum'], m.hexdigest())) + (fileinfo['filename'], fileinfo['checksum'], + m.hexdigest())) fileinfo['hub.checked_md5'] = True if fileinfo['buildroot_id'] not in self.br_prep: - raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) + raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % + fileinfo) if fileinfo['type'] not in ['rpm', 'log']: self.prep_archive(fileinfo) if fileinfo['type'] == 'rpm': @@ -6992,7 +7074,8 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No be any non-rpm filetype supported by Koji. filepath: full path to the archive file - buildinfo: dict of information about the build to associate the archive with (as returned by getBuild()) + buildinfo: dict of information about the build to associate the archive with + (as returned by getBuild()) type: type of the archive being imported. Currently supported archive types: maven, win, image typeInfo: dict of type-specific information buildroot_id: the id of the buildroot the archive was built in (may be None) @@ -7081,14 +7164,16 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No pom_maveninfo = koji.pom_to_maven_info(pom_info) # sanity check: Maven info from pom must match the user-supplied typeInfo if koji.mavenLabel(pom_maveninfo) != koji.mavenLabel(typeInfo): - raise koji.BuildError('Maven info from .pom file (%s) does not match user-supplied typeInfo (%s)' % - (koji.mavenLabel(pom_maveninfo), koji.mavenLabel(typeInfo))) + raise koji.BuildError( + 'Maven info from .pom file (%s) does not match user-supplied typeInfo (%s)' % + (koji.mavenLabel(pom_maveninfo), koji.mavenLabel(typeInfo))) # sanity check: the filename of the pom file must match -.pom if filename != '%(artifact_id)s-%(version)s.pom' % typeInfo: raise koji.BuildError('Maven info (%s) is not consistent with pom filename (%s)' % (koji.mavenLabel(typeInfo), filename)) - insert = InsertProcessor('maven_archives', data=dslice(typeInfo, ('group_id', 'artifact_id', 'version'))) + insert = InsertProcessor('maven_archives', + data=dslice(typeInfo, ('group_id', 'artifact_id', 'version'))) insert.set(archive_id=archive_id) insert.execute() @@ -7151,7 +7236,8 @@ def _import_archive_file(filepath, destdir): if os.path.exists(final_path): raise koji.GenericError("Error importing archive file, %s already exists" % final_path) if os.path.islink(filepath) or not os.path.isfile(filepath): - raise koji.GenericError("Error importing archive file, %s is not a regular file" % filepath) + raise koji.GenericError("Error importing archive file, %s is not a regular file" % + filepath) move_and_symlink(filepath, final_path, create_dir=True) @@ -7234,7 +7320,8 @@ def add_rpm_sig(an_rpm, sighdr): koji.ensuredir(os.path.dirname(sigpath)) with open(sigpath, 'wb') as fo: fo.write(sighdr) - koji.plugin.run_callbacks('postRPMSign', sigkey=sigkey, sighash=sighash, build=binfo, rpm=rinfo) + koji.plugin.run_callbacks('postRPMSign', + sigkey=sigkey, sighash=sighash, build=binfo, rpm=rinfo) def _scan_sighdr(sighdr, fn): @@ -7411,8 +7498,10 @@ def query_history(tables=None, **kwargs): 'user_perms': ['user_id', 'perm_id'], 'user_groups': ['user_id', 'group_id'], 'cg_users': ['user_id', 'cg_id'], - 'tag_inheritance': ['tag_id', 'parent_id', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter'], - 'tag_config': ['tag_id', 'arches', 'perm_id', 'locked', 'maven_support', 'maven_include_all'], + 'tag_inheritance': ['tag_id', 'parent_id', 'priority', 'maxdepth', 'intransitive', + 'noconfig', 'pkg_filter'], + 'tag_config': ['tag_id', 'arches', 'perm_id', 'locked', 'maven_support', + 'maven_include_all'], 'tag_extra': ['tag_id', 'key', 'value'], 'build_target_config': ['build_target_id', 'build_tag', 'dest_tag'], 'external_repo_config': ['external_repo_id', 'url'], @@ -7422,10 +7511,11 @@ def query_history(tables=None, **kwargs): 'tag_listing': ['build_id', 'tag_id'], 'tag_packages': ['package_id', 'tag_id', 'blocked', 'extra_arches'], 'tag_package_owners': ['package_id', 'tag_id', 'owner'], - 'group_config': ['group_id', 'tag_id', 'blocked', 'exported', 'display_name', 'is_default', 'uservisible', - 'description', 'langonly', 'biarchonly'], + 'group_config': ['group_id', 'tag_id', 'blocked', 'exported', 'display_name', 'is_default', + 'uservisible', 'description', 'langonly', 'biarchonly'], 'group_req_listing': ['group_id', 'tag_id', 'req_id', 'blocked', 'type', 'is_metapkg'], - 'group_package_listing': ['group_id', 'tag_id', 'package', 'blocked', 'type', 'basearchonly', 'requires'], + 'group_package_listing': ['group_id', 'tag_id', 'package', 'blocked', 'type', + 'basearchonly', 'requires'], } name_joins = { # joins triggered by table fields for name lookup @@ -7480,7 +7570,8 @@ def query_history(tables=None, **kwargs): if join_as == tbl: joins.append('LEFT OUTER JOIN %s ON %s = %s.id' % (tbl, field, tbl)) else: - joins.append('LEFT OUTER JOIN %s AS %s ON %s = %s.id' % (tbl, join_as, field, join_as)) + joins.append('LEFT OUTER JOIN %s AS %s ON %s = %s.id' % + (tbl, join_as, field, join_as)) elif field == 'build_id': # special case fields.update({ @@ -7603,7 +7694,8 @@ def query_history(tables=None, **kwargs): clauses.append('ev1.time > %(after)s OR ev2.time > %(after)s') fields['ev1.time > %(after)s'] = '_created_after' fields['ev2.time > %(after)s'] = '_revoked_after' - # clauses.append('EXTRACT(EPOCH FROM ev1.time) > %(after)s OR EXTRACT(EPOCH FROM ev2.time) > %(after)s') + # clauses.append('EXTRACT(EPOCH FROM ev1.time) > %(after)s OR ' + # 'EXTRACT(EPOCH FROM ev2.time) > %(after)s') elif arg == 'afterEvent': data['afterEvent'] = value c_test = '%s.create_event > %%(afterEvent)i' % table @@ -7616,7 +7708,8 @@ def query_history(tables=None, **kwargs): value = datetime.datetime.fromtimestamp(value).isoformat(' ') data['before'] = value clauses.append('ev1.time < %(before)s OR ev2.time < %(before)s') - # clauses.append('EXTRACT(EPOCH FROM ev1.time) < %(before)s OR EXTRACT(EPOCH FROM ev2.time) < %(before)s') + # clauses.append('EXTRACT(EPOCH FROM ev1.time) < %(before)s OR ' + # 'EXTRACT(EPOCH FROM ev2.time) < %(before)s') fields['ev1.time < %(before)s'] = '_created_before' fields['ev2.time < %(before)s'] = '_revoked_before' elif arg == 'beforeEvent': @@ -7750,7 +7843,8 @@ def build_references(build_id, limit=None, lazy=False): st_complete = koji.BUILD_STATES['COMPLETE'] fields = ('id', 'name', 'version', 'release', 'arch', 'build_id') idx = {} - q = """SELECT rpminfo.id, rpminfo.name, rpminfo.version, rpminfo.release, rpminfo.arch, rpminfo.build_id + q = """SELECT + rpminfo.id, rpminfo.name, rpminfo.version, rpminfo.release, rpminfo.arch, rpminfo.build_id FROM rpminfo, build WHERE rpminfo.buildroot_id IN ( @@ -7795,7 +7889,8 @@ def build_references(build_id, limit=None, lazy=False): # find archives whose buildroots we were in fields = ('id', 'type_id', 'type_name', 'build_id', 'filename') idx = {} - q = """SELECT archiveinfo.id, archiveinfo.type_id, archivetypes.name, archiveinfo.build_id, archiveinfo.filename + q = """SELECT archiveinfo.id, archiveinfo.type_id, archivetypes.name, archiveinfo.build_id, + archiveinfo.filename FROM buildroot_archives JOIN archiveinfo ON archiveinfo.buildroot_id = buildroot_archives.buildroot_id JOIN build ON archiveinfo.build_id = build.id @@ -7897,11 +7992,13 @@ def delete_build(build, strict=True, min_ref_age=604800): return False if refs.get('archives'): if strict: - raise koji.GenericError("Cannot delete build, used in archive buildroots: %s" % refs['archives']) + raise koji.GenericError("Cannot delete build, used in archive buildroots: %s" % + refs['archives']) return False if refs.get('component_of'): if strict: - raise koji.GenericError("Cannot delete build, used as component of: %r" % refs['component_of']) + raise koji.GenericError("Cannot delete build, used as component of: %r" % + refs['component_of']) return False if refs.get('last_used'): age = time.time() - refs['last_used'] @@ -7935,7 +8032,8 @@ def _delete_build(binfo): # files on disk: DELETE st_deleted = koji.BUILD_STATES['DELETED'] st_old = binfo['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_deleted, info=binfo) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=st_deleted, info=binfo) build_id = binfo['id'] q = """SELECT id FROM rpminfo WHERE build_id=%(build_id)i""" rpm_ids = _fetchMulti(q, locals()) @@ -7952,7 +8050,8 @@ def _delete_build(binfo): if os.path.exists(builddir): koji.util.rmtree(builddir) binfo = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_deleted, info=binfo) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=st_deleted, info=binfo) def reset_build(build): @@ -7973,7 +8072,9 @@ def reset_build(build): # nothing to do return st_old = binfo['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=koji.BUILD_STATES['CANCELED'], info=binfo) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=koji.BUILD_STATES['CANCELED'], + info=binfo) q = """SELECT id FROM rpminfo WHERE build_id=%(id)i""" ids = _fetchMulti(q, binfo) for (rpm_id,) in ids: @@ -8022,7 +8123,9 @@ def reset_build(build): if os.path.exists(builddir): koji.util.rmtree(builddir) binfo = get_build(build, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=koji.BUILD_STATES['CANCELED'], info=binfo) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=koji.BUILD_STATES['CANCELED'], + info=binfo) def cancel_build(build_id, cancel_task=True): @@ -8043,7 +8146,8 @@ def cancel_build(build_id, cancel_task=True): if build['state'] != st_building: return False st_old = build['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_canceled, info=build) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=st_canceled, info=build) update = """UPDATE build SET state = %(st_canceled)i, completion_time = NOW() WHERE id = %(build_id)i AND state = %(st_building)i""" @@ -8062,7 +8166,8 @@ def cancel_build(build_id, cancel_task=True): _dml(delete, {'build_id': build_id}) build = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_canceled, info=build) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=st_canceled, info=build) return True @@ -8179,7 +8284,8 @@ def get_notification_recipients(build, tag_id, state): return list(set(emails)) -def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_success=False, failure_msg=''): +def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_success=False, + failure_msg=''): if context.opts.get('DisableNotifications'): return if is_successful: @@ -8203,7 +8309,9 @@ def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_s recipients[email] = 1 recipients_uniq = to_list(recipients.keys()) if len(recipients_uniq) > 0 and not (is_successful and ignore_success): - task_id = make_task('tagNotification', [recipients_uniq, is_successful, tag_id, from_id, build_id, user_id, ignore_success, failure_msg]) + task_id = make_task('tagNotification', + [recipients_uniq, is_successful, tag_id, from_id, build_id, user_id, + ignore_success, failure_msg]) return task_id return None @@ -8842,8 +8950,10 @@ SELECT %(col_str)s return query def __repr__(self): - return '' % \ - (self.columns, self.aliases, self.tables, self.joins, self.clauses, self.values, self.opts) + return '' % \ + (self.columns, self.aliases, self.tables, self.joins, self.clauses, self.values, + self.opts) def _seqtostr(self, seq, sep=', ', sort=False): if seq: @@ -9580,7 +9690,9 @@ def check_policy(name, data, default='deny', strict=False): reason = reason.lower() lastrule = ruleset.last_rule() if context.opts.get('KojiDebug', False): - logger.error("policy %(name)s gave %(result)s, reason: %(reason)s, last rule: %(lastrule)s", locals()) + logger.error( + "policy %(name)s gave %(result)s, reason: %(reason)s, last rule: %(lastrule)s", + locals()) if result == 'allow': return True, reason if result != 'deny': @@ -9684,7 +9796,8 @@ def importImageInternal(task_id, build_id, imgdata): if os.path.exists(final_path): raise koji.GenericError("Error importing build log. %s already exists." % final_path) if os.path.islink(logsrc) or not os.path.isfile(logsrc): - raise koji.GenericError("Error importing build log. %s is not a regular file." % logsrc) + raise koji.GenericError("Error importing build log. %s is not a regular file." % + logsrc) move_and_symlink(logsrc, final_path, create_dir=True) # record all of the RPMs installed in the image(s) @@ -9812,8 +9925,9 @@ class RootExports(object): build: The build to generate wrapper rpms for. Must be in the COMPLETE state and have no rpms already associated with it. url: SCM URL to a specfile fragment - target: The build target to use when building the wrapper rpm. The build_tag of the target will - be used to populate the buildroot in which the rpms are built. + target: The build target to use when building the wrapper rpm. + The build_tag of the target will be used to populate the buildroot in which the + rpms are built. priority: the amount to increase (or decrease) the task priority, relative to the default priority; higher values mean lower priority; only admins have the right to specify a negative priority here @@ -9830,7 +9944,8 @@ class RootExports(object): build = self.getBuild(build, strict=True) if list_rpms(build['id']) and not (opts.get('scratch') or opts.get('create_build')): - raise koji.PreBuildError('wrapper rpms for %s have already been built' % koji.buildLabel(build)) + raise koji.PreBuildError('wrapper rpms for %s have already been built' % + koji.buildLabel(build)) build_target = self.getBuildTarget(target) if not build_target: raise koji.PreBuildError('no such build target: %s' % target) @@ -9951,7 +10066,8 @@ class RootExports(object): taskOpts['priority'] = koji.PRIO_DEFAULT + priority if 'scratch' not in opts and 'indirection_template_url' not in opts: - raise koji.ActionNotAllowed('Non-scratch builds must provide url for the indirection template') + raise koji.ActionNotAllowed( + 'Non-scratch builds must provide url for the indirection template') if 'arch' in opts: taskOpts['arch'] = opts['arch'] @@ -10212,7 +10328,9 @@ class RootExports(object): given ID.""" if '..' in fileName: raise koji.GenericError('Invalid file name: %s' % fileName) - filePath = '%s/%s/%s' % (koji.pathinfo.work(volume), koji.pathinfo.taskrelpath(taskID), fileName) + filePath = '%s/%s/%s' % (koji.pathinfo.work(volume), + koji.pathinfo.taskrelpath(taskID), + fileName) filePath = os.path.normpath(filePath) if not os.path.isfile(filePath): raise koji.GenericError('no file "%s" output by task %i' % (fileName, taskID)) @@ -10264,7 +10382,8 @@ class RootExports(object): filepath: path to the archive file (relative to the Koji workdir) buildinfo: information about the build to associate the archive with - May be a string (NVR), integer (buildID), or dict (containing keys: name, version, release) + May be a string (NVR), integer (buildID), or dict (containing keys: name, + version, release) type: type of the archive being imported. Currently supported archive types: maven, win typeInfo: dict of type-specific information """ @@ -10602,12 +10721,16 @@ class RootExports(object): for build in build_list: policy_data['build'] = build['id'] assert_policy('tag', policy_data) - # XXX - we're running this check twice, here and in host.tagBuild (called by the task) + # XXX - we're running this check twice, here and in host.tagBuild (called by the + # task) wait_on = [] tasklist = [] for build in build_list: - task_id = make_task('dependantTask', [wait_on, [['tagBuild', [tag2_id, build['id'], force, tag1_id], {'priority': 15}]]]) + task_id = make_task('dependantTask', + [wait_on, [['tagBuild', + [tag2_id, build['id'], force, tag1_id], + {'priority': 15}]]]) wait_on = [task_id] log_error("\nMade Task: %s\n" % task_id) tasklist.append(task_id) @@ -10644,11 +10767,11 @@ class RootExports(object): - author: only return changelogs with a matching author - before: only return changelogs from before the given date (in UTC) - (a datetime object, a string in the 'YYYY-MM-DD HH24:MI:SS format, or integer seconds - since the epoch) + (a datetime object, a string in the 'YYYY-MM-DD HH24:MI:SS format, or integer + seconds since the epoch) - after: only return changelogs from after the given date (in UTC) - (a datetime object, a string in the 'YYYY-MM-DD HH24:MI:SS format, or integer seconds - since the epoch) + (a datetime object, a string in the 'YYYY-MM-DD HH24:MI:SS format, or integer + seconds since the epoch) - queryOpts: query options used by the QueryProcessor - strict: if srpm doesn't exist raise an error, otherwise return empty list @@ -10716,7 +10839,8 @@ class RootExports(object): results = [] - fields = koji.get_header_fields(srpm_path, ['changelogtime', 'changelogname', 'changelogtext']) + fields = koji.get_header_fields(srpm_path, + ['changelogtime', 'changelogname', 'changelogtext']) for (cltime, clname, cltext) in zip(fields['changelogtime'], fields['changelogname'], fields['changelogtext']): cldate = datetime.datetime.fromtimestamp(cltime).isoformat(' ') @@ -10733,7 +10857,10 @@ class RootExports(object): if queryOpts.get('asList'): results.append([cldate, clname, cltext]) else: - results.append({'date': cldate, 'date_ts': cltime, 'author': clname, 'text': cltext}) + results.append({'date': cldate, + 'date_ts': cltime, + 'author': clname, + 'text': cltext}) results = _applyQueryOpts(results, queryOpts) return koji.fixEncodingRecurse(results, remove_nonprintable=True) @@ -10800,32 +10927,40 @@ class RootExports(object): raise koji.GenericError("Finished task's priority can't be updated") task.setPriority(priority, recurse=recurse) - def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, owner=None, type=None): + def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, + owner=None, type=None): """List builds tagged with tag""" # lookup tag id tag = get_tag(tag, strict=True, event=event)['id'] - results = readTaggedBuilds(tag, event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) + results = readTaggedBuilds(tag, event, inherit=inherit, latest=latest, package=package, + owner=owner, type=type) if prefix: prefix = prefix.lower() - results = [build for build in results if build['package_name'].lower().startswith(prefix)] + results = [build for build in results + if build['package_name'].lower().startswith(prefix)] return results - def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, rpmsigs=False, owner=None, type=None): + def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, + rpmsigs=False, owner=None, type=None): """List rpms and builds within tag""" # lookup tag id tag = get_tag(tag, strict=True, event=event)['id'] - return readTaggedRPMS(tag, event=event, inherit=inherit, latest=latest, package=package, arch=arch, rpmsigs=rpmsigs, owner=owner, type=type) + return readTaggedRPMS(tag, event=event, inherit=inherit, latest=latest, package=package, + arch=arch, rpmsigs=rpmsigs, owner=owner, type=type) - def listTaggedArchives(self, tag, event=None, inherit=False, latest=False, package=None, type=None): + def listTaggedArchives(self, tag, event=None, inherit=False, latest=False, package=None, + type=None): """List archives and builds within a tag""" # lookup tag id tag = get_tag(tag, strict=True, event=event)['id'] - return readTaggedArchives(tag, event=event, inherit=inherit, latest=latest, package=package, type=type) + return readTaggedArchives(tag, event=event, inherit=inherit, latest=latest, + package=package, type=type) def listBuilds(self, packageID=None, userID=None, taskID=None, prefix=None, state=None, volumeID=None, source=None, createdBefore=None, createdAfter=None, - completeBefore=None, completeAfter=None, type=None, typeInfo=None, queryOpts=None): + completeBefore=None, completeAfter=None, type=None, typeInfo=None, + queryOpts=None): """Return a list of builds that match the given parameters Filter parameters @@ -10899,16 +11034,20 @@ class RootExports(object): If no builds match, an empty list is returned. """ - fields = [('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), - ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + fields = [('build.id', 'build_id'), ('build.version', 'version'), + ('build.release', 'release'), + ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.source', 'source'), ('build.extra', 'extra'), - ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('build.task_id', 'task_id'), + ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), + ('build.task_id', 'task_id'), ('EXTRACT(EPOCH FROM events.time)', 'creation_ts'), ('EXTRACT(EPOCH FROM build.start_time)', 'start_ts'), ('EXTRACT(EPOCH FROM build.completion_time)', 'completion_ts'), - ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), + ('package.id', 'package_id'), ('package.name', 'package_name'), + ('package.name', 'name'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), ('users.id', 'owner_id'), ('users.name', 'owner_name')] @@ -11007,7 +11146,8 @@ class RootExports(object): if not isinstance(tag, six.integer_types): # lookup tag id tag = get_tag_id(tag, strict=True) - return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, latest=True, rpmsigs=rpmsigs, type=type) + return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, + latest=True, rpmsigs=rpmsigs, type=type) def getLatestMavenArchives(self, tag, event=None, inherit=True): """Return a list of the latest Maven archives in the tag, as of the given event @@ -11163,7 +11303,8 @@ class RootExports(object): results = [] - for dep_name in ['REQUIRE', 'PROVIDE', 'CONFLICT', 'OBSOLETE', 'SUGGEST', 'ENHANCE', 'SUPPLEMENT', 'RECOMMEND']: + for dep_name in ['REQUIRE', 'PROVIDE', 'CONFLICT', 'OBSOLETE', 'SUGGEST', 'ENHANCE', + 'SUPPLEMENT', 'RECOMMEND']: dep_id = getattr(koji, 'DEP_' + dep_name) if depType is None or depType == dep_id: fields = koji.get_header_fields(rpm_path, [dep_name + 'NAME', @@ -11175,7 +11316,8 @@ class RootExports(object): if queryOpts.get('asList'): results.append([name, version, flags, dep_id]) else: - results.append({'name': name, 'version': version, 'flags': flags, 'type': dep_id}) + results.append( + {'name': name, 'version': version, 'flags': flags, 'type': dep_id}) return _applyQueryOpts(results, queryOpts) @@ -11204,13 +11346,15 @@ class RootExports(object): results = [] hdr = koji.get_rpm_header(rpm_path) fields = koji.get_header_fields(hdr, ['filenames', 'filemd5s', 'filesizes', 'fileflags', - 'fileusername', 'filegroupname', 'filemtimes', 'filemodes']) + 'fileusername', 'filegroupname', 'filemtimes', + 'filemodes']) digest_algo = koji.util.filedigestAlgo(hdr) - for (name, digest, size, flags, user, group, mtime, mode) in zip(fields['filenames'], fields['filemd5s'], - fields['filesizes'], fields['fileflags'], - fields['fileusername'], fields['filegroupname'], - fields['filemtimes'], fields['filemodes']): + for (name, digest, size, flags, user, group, mtime, mode) \ + in zip(fields['filenames'], fields['filemd5s'], + fields['filesizes'], fields['fileflags'], + fields['fileusername'], fields['filegroupname'], + fields['filemtimes'], fields['filemodes']): if queryOpts.get('asList'): results.append([name, digest, size, flags, digest_algo, user, group, mtime, mode]) else: @@ -11261,7 +11405,8 @@ class RootExports(object): hdr = koji.get_rpm_header(rpm_path) # use filemd5s for backward compatibility fields = koji.get_header_fields(hdr, ['filenames', 'filemd5s', 'filesizes', 'fileflags', - 'fileusername', 'filegroupname', 'filemtimes', 'filemodes']) + 'fileusername', 'filegroupname', 'filemtimes', + 'filemodes']) digest_algo = koji.util.filedigestAlgo(hdr) i = 0 @@ -11347,7 +11492,8 @@ class RootExports(object): getPackage = staticmethod(lookup_package) - def listPackages(self, tagID=None, userID=None, pkgID=None, prefix=None, inherited=False, with_dups=False, event=None, queryOpts=None): + def listPackages(self, tagID=None, userID=None, pkgID=None, prefix=None, inherited=False, + with_dups=False, event=None, queryOpts=None): """List if tagID and/or userID is specified, limit the list to packages belonging to the given user or with the given tag. @@ -11391,7 +11537,8 @@ class RootExports(object): if prefix: prefix = prefix.lower() - results = [package for package in results if package['package_name'].lower().startswith(prefix)] + results = [package for package in results + if package['package_name'].lower().startswith(prefix)] return _applyQueryOpts(results, queryOpts) @@ -11452,7 +11599,8 @@ class RootExports(object): perm = lookup_perm(permission, strict=(not create), create=create) perm_id = perm['id'] if perm['name'] in koji.auth.get_user_perms(user_id): - raise koji.GenericError('user %s already has permission: %s' % (userinfo, perm['name'])) + raise koji.GenericError('user %s already has permission: %s' % + (userinfo, perm['name'])) insert = InsertProcessor('user_perms') insert.set(user_id=user_id, perm_id=perm_id) insert.make_create() @@ -11465,7 +11613,8 @@ class RootExports(object): perm = lookup_perm(permission, strict=True) perm_id = perm['id'] if perm['name'] not in koji.auth.get_user_perms(user_id): - raise koji.GenericError('user %s does not have permission: %s' % (userinfo, perm['name'])) + raise koji.GenericError('user %s does not have permission: %s' % + (userinfo, perm['name'])) update = UpdateProcessor('user_perms', values=locals(), clauses=["user_id = %(user_id)i", "perm_id = %(perm_id)i"]) update.make_revoke() @@ -11584,7 +11733,8 @@ class RootExports(object): else: id = get_tag_id(tag, strict=True) - fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)', 'repo.dist'] + fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', + 'EXTRACT(EPOCH FROM events.time)', 'repo.dist'] aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts', 'dist'] joins = ['events ON repo.create_event = events.id'] clauses = ['repo.tag_id = %(id)i'] @@ -11628,7 +11778,8 @@ class RootExports(object): for task_id in task_ids: logger.debug("Cancelling distRepo task %d" % task_id) Task(task_id).cancel(recurse=True) - return make_task('distRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') + return make_task('distRepo', [tag, repo_id, keys, task_opts], + priority=15, channel='createrepo') def newRepo(self, tag, event=None, src=False, debuginfo=False, separate_src=False): """Create a newRepo task. returns task id""" @@ -11758,7 +11909,8 @@ class RootExports(object): owner[int|list]: limit to tasks owned by the user with the given ID not_owner[int|list]: limit to tasks not owned by the user with the given ID host_id[int|list]: limit to tasks running on the host with the given ID - not_host_id[int|list]: limit to tasks running on the hosts with IDs other than the given ID + not_host_id[int|list]: limit to tasks running on the hosts with IDs other than the + given ID channel_id[int|list]: limit to tasks in the specified channel not_channel_id[int|list]: limit to tasks not in the specified channel parent[int|list]: limit to tasks with the given parent @@ -11954,7 +12106,9 @@ class RootExports(object): args = task.getRequest() channel = get_channel(taskInfo['channel_id'], strict=True) - return make_task(taskInfo['method'], args, arch=taskInfo['arch'], channel=channel['name'], priority=taskInfo['priority']) + return make_task(taskInfo['method'], args, + arch=taskInfo['arch'], channel=channel['name'], + priority=taskInfo['priority']) def addHost(self, hostname, arches, krb_principal=None): """ @@ -11986,7 +12140,8 @@ class RootExports(object): krb_principal=krb_principal) # host entry hostID = _singleValue("SELECT nextval('host_id_seq')", strict=True) - insert = "INSERT INTO host (id, user_id, name) VALUES (%(hostID)i, %(userID)i, %(hostname)s)" + insert = "INSERT INTO host (id, user_id, name) VALUES (%(hostID)i, %(userID)i, " \ + "%(hostname)s)" _dml(insert, dslice(locals(), ('hostID', 'userID', 'hostname'))) insert = InsertProcessor('host_config') @@ -12017,7 +12172,8 @@ class RootExports(object): renameChannel = staticmethod(rename_channel) removeChannel = staticmethod(remove_channel) - def listHosts(self, arches=None, channelID=None, ready=None, enabled=None, userID=None, queryOpts=None): + def listHosts(self, arches=None, channelID=None, ready=None, enabled=None, userID=None, + queryOpts=None): """Get a list of hosts. "arches" is a list of string architecture names, e.g. ['i386', 'ppc64']. If one of the arches associated with a given host appears in the list, it will be included in the results. If "ready" and "enabled" @@ -12152,11 +12308,15 @@ class RootExports(object): userid = userinfo['id'] buildid = buildinfo['id'] owner_id_old = buildinfo['owner_id'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='owner_id', old=owner_id_old, new=userid, info=buildinfo) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='owner_id', old=owner_id_old, new=userid, + info=buildinfo) q = """UPDATE build SET owner=%(userid)i WHERE id=%(buildid)i""" _dml(q, locals()) buildinfo = get_build(build, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='owner_id', old=owner_id_old, new=userid, info=buildinfo) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='owner_id', old=owner_id_old, new=userid, + info=buildinfo) def setBuildTimestamp(self, build, ts): """Set the completion time for a build @@ -12176,20 +12336,23 @@ class RootExports(object): elif not isinstance(ts, NUMERIC_TYPES): raise koji.GenericError("Invalid type for timestamp") ts_old = buildinfo['completion_ts'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) buildid = buildinfo['id'] q = """UPDATE build SET completion_time=TIMESTAMP 'epoch' AT TIME ZONE 'utc' + '%(ts)f seconds'::interval WHERE id=%%(buildid)i""" % locals() _dml(q, locals()) buildinfo = get_build(build, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) def count(self, methodName, *args, **kw): """Execute the XML-RPC method with the given name and count the results. - A method return value of None will return O, a return value of type "list", "tuple", or "dict" - will return len(value), and a return value of any other type will return 1. An invalid - methodName will raise an AttributeError, and invalid arguments will raise a TypeError.""" + A method return value of None will return O, a return value of type "list", "tuple", or + "dict" will return len(value), and a return value of any other type will return 1. An + invalid methodName will raise an AttributeError, and invalid arguments will raise a + TypeError.""" result = getattr(self, methodName)(*args, **kw) if result is None: return 0 @@ -12463,7 +12626,8 @@ class RootExports(object): be replaced with "%". If matchType is "regexp", no changes will be made.""" if matchType == 'glob': - return terms.replace('\\', '\\\\').replace('_', r'\_').replace('?', '_').replace('*', '%') + return terms.replace( + '\\', '\\\\').replace('_', r'\_').replace('?', '_').replace('*', '%') else: return terms @@ -12523,24 +12687,29 @@ class RootExports(object): joins = [] if type == 'build': joins.append('package ON build.pkg_id = package.id') - clause = "package.name || '-' || build.version || '-' || build.release %s %%(terms)s" % oper + clause = "package.name || '-' || build.version || '-' || build.release %s %%(terms)s" \ + % oper cols = ('build.id', "package.name || '-' || build.version || '-' || build.release") elif type == 'rpm': - clause = "name || '-' || version || '-' || release || '.' || arch || '.rpm' %s %%(terms)s" % oper + clause = "name || '-' || version || '-' || release || '.' || arch || '.rpm' %s " \ + "%%(terms)s" % oper cols = ('id', "name || '-' || version || '-' || release || '.' || arch || '.rpm'") elif type == 'tag': joins.append('tag_config ON tag.id = tag_config.tag_id') clause = 'tag_config.active = TRUE and name %s %%(terms)s' % oper elif type == 'target': - joins.append('build_target_config ON build_target.id = build_target_config.build_target_id') + joins.append('build_target_config ' + 'ON build_target.id = build_target_config.build_target_id') clause = 'build_target_config.active = TRUE and name %s %%(terms)s' % oper elif type == 'maven': cols = ('id', 'filename') joins.append('maven_archives ON archiveinfo.id = maven_archives.archive_id') clause = "archiveinfo.filename %s %%(terms)s or maven_archives.group_id || '-' || " \ - "maven_archives.artifact_id || '-' || maven_archives.version %s %%(terms)s" % (oper, oper) + "maven_archives.artifact_id || '-' || maven_archives.version %s %%(terms)s" \ + % (oper, oper) elif type == 'win': - cols = ('id', "trim(leading '/' from win_archives.relpath || '/' || archiveinfo.filename)") + cols = ('id', + "trim(leading '/' from win_archives.relpath || '/' || archiveinfo.filename)") joins.append('win_archives ON archiveinfo.id = win_archives.archive_id') clause = "archiveinfo.filename %s %%(terms)s or win_archives.relpath || '/' || " \ "archiveinfo.filename %s %%(terms)s" % (oper, oper) @@ -12725,7 +12894,8 @@ class BuildRoot(object): ) query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], tables=['buildroot_listing'], - joins=["rpminfo ON rpm_id = rpminfo.id", "external_repo ON external_repo_id = external_repo.id"], + joins=["rpminfo ON rpm_id = rpminfo.id", + "external_repo ON external_repo_id = external_repo.id"], clauses=["buildroot_listing.buildroot_id = %(brootid)i"], values=locals()) return query.execute() @@ -12882,12 +13052,15 @@ class Host(object): update.execute() elif tasks: # wait on specified subtasks - update = UpdateProcessor('task', clauses=['id IN %(tasks)s', 'parent=%(parent)s'], values=locals()) + update = UpdateProcessor('task', clauses=['id IN %(tasks)s', 'parent=%(parent)s'], + values=locals()) update.set(awaited=True) update.execute() # clear awaited flag on any other child tasks update = UpdateProcessor('task', values=locals(), - clauses=['id NOT IN %(tasks)s', 'parent=%(parent)s', 'awaited=true']) + clauses=['id NOT IN %(tasks)s', + 'parent=%(parent)s', + 'awaited=true']) update.set(awaited=False) update.execute() else: @@ -13360,7 +13533,8 @@ class HostExports(object): st_old = build_info['state'] st_complete = koji.BUILD_STATES['COMPLETE'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=build_info) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=st_complete, info=build_info) update = UpdateProcessor('build', clauses=['id=%(build_id)i'], values={'build_id': build_id}) @@ -13370,7 +13544,8 @@ class HostExports(object): update.set(volume_id=build_info['volume_id']) update.execute() build_info = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_complete, info=build_info) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=st_complete, info=build_info) # send email build_notification(task_id, build_id) @@ -13487,7 +13662,8 @@ class HostExports(object): # update build state st_complete = koji.BUILD_STATES['COMPLETE'] st_old = build_info['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=build_info) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=st_complete, info=build_info) update = UpdateProcessor('build', clauses=['id=%(build_id)i'], values={'build_id': build_id}) update.set(state=st_complete) @@ -13496,7 +13672,8 @@ class HostExports(object): update.rawset(completion_time='now()') update.execute() build_info = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_complete, info=build_info) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=st_complete, info=build_info) # send email build_notification(task_id, build_id) @@ -13531,12 +13708,14 @@ class HostExports(object): build_info = get_build(build_id, strict=True) if build_info['state'] != koji.BUILD_STATES['COMPLETE']: - raise koji.GenericError('cannot import wrapper rpms for %s: build state is %s, not complete' % - (koji.buildLabel(build_info), koji.BUILD_STATES[build_info['state']].lower())) + raise koji.GenericError( + 'cannot import wrapper rpms for %s: build state is %s, not complete' % + (koji.buildLabel(build_info), koji.BUILD_STATES[build_info['state']].lower())) if list_rpms(buildID=build_info['id']): # don't allow overwriting of already-imported wrapper RPMs - raise koji.GenericError('wrapper rpms for %s have already been imported' % koji.buildLabel(build_info)) + raise koji.GenericError('wrapper rpms for %s have already been imported' % + koji.buildLabel(build_info)) _import_wrapper(task.id, build_info, rpm_results) @@ -13613,7 +13792,8 @@ class HostExports(object): raise koji.BuildError('unsupported file type: %s' % relpath) filepath = joinpath(task_dir, relpath) metadata['relpath'] = os.path.dirname(relpath) - import_archive(filepath, build_info, 'win', metadata, buildroot_id=results['buildroot_id']) + import_archive(filepath, build_info, 'win', metadata, + buildroot_id=results['buildroot_id']) # move the logs to their final destination for relpath in results['logs']: @@ -13632,7 +13812,8 @@ class HostExports(object): # update build state st_old = build_info['state'] st_complete = koji.BUILD_STATES['COMPLETE'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=build_info) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=st_complete, info=build_info) update = UpdateProcessor('build', clauses=['id=%(build_id)i'], values={'build_id': build_id}) update.set(state=st_complete) @@ -13641,7 +13822,8 @@ class HostExports(object): update.rawset(completion_time='now()') update.execute() build_info = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_complete, info=build_info) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=st_complete, info=build_info) # send email build_notification(task_id, build_id) @@ -13658,7 +13840,8 @@ class HostExports(object): st_failed = koji.BUILD_STATES['FAILED'] buildinfo = get_build(build_id, strict=True) st_old = buildinfo['state'] - koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_failed, info=buildinfo) + koji.plugin.run_callbacks('preBuildStateChange', + attribute='state', old=st_old, new=st_failed, info=buildinfo) query = """SELECT state, completion_time FROM build @@ -13679,7 +13862,8 @@ class HostExports(object): WHERE id = %(build_id)i""" _dml(update, locals()) buildinfo = get_build(build_id, strict=True) - koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_failed, info=buildinfo) + koji.plugin.run_callbacks('postBuildStateChange', + attribute='state', old=st_old, new=st_failed, info=buildinfo) build_notification(task_id, build_id) def tagBuild(self, task_id, tag, build, force=False, fromtag=None): @@ -13742,12 +13926,14 @@ class HostExports(object): _import_wrapper(rpm_results['task_id'], get_build(build_id, strict=True), rpm_results) - def tagNotification(self, is_successful, tag_id, from_id, build_id, user_id, ignore_success=False, failure_msg=''): + def tagNotification(self, is_successful, tag_id, from_id, build_id, user_id, + ignore_success=False, failure_msg=''): """Create a tag notification message. Handles creation of tagNotification tasks for hosts.""" host = Host() host.verify() - tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_success, failure_msg) + tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_success, + failure_msg) def checkPolicy(self, name, data, default='deny', strict=False): host = Host() @@ -13845,7 +14031,9 @@ class HostExports(object): archive['artifact_id'], {}).setdefault( archive['version'], archive['build_id']) if idx_build != archive['build_id']: - logger.error("Found multiple builds for %(group_id)s:%(artifact_id)s:%(version)s. Current build: %(build_id)i", archive) + logger.error( + "Found multiple builds for %(group_id)s:%(artifact_id)s:%(version)s. " + "Current build: %(build_id)i", archive) logger.error("Indexed build id was %i", idx_build) if not ignore: @@ -13885,9 +14073,19 @@ class HostExports(object): archive['artifact_id'], {}).setdefault( archive['version'], archive['build_id']) if idx_build != archive['build_id']: - logger.error("Overriding build for %(group_id)s:%(artifact_id)s:%(version)s.", archive) - logger.error("Current build is %s, new build is %s.", idx_build, archive['build_id']) - maven_build_index[archive['group_id']][archive['artifact_id']][archive['version']] = archive['build_id'] + logger.error( + "Overriding build for %(group_id)s:%(artifact_id)s:%(version)s.", + archive) + logger.error( + "Current build is %s, new build is %s.", + idx_build, archive['build_id']) + maven_build_index[ + archive['group_id'] + ][ + archive['artifact_id'] + ][ + archive['version'] + ] = archive['build_id'] ignore.extend(task_deps.values()) @@ -13938,23 +14136,29 @@ class HostExports(object): pass else: if not ignore_unknown: - logger.error("Unknown file for %(group_id)s:%(artifact_id)s:%(version)s", maven_info) + logger.error("Unknown file for %(group_id)s:%(artifact_id)s:%(version)s", + maven_info) if build_id: build = get_build(build_id) logger.error("g:a:v supplied by build %(nvr)s", build) - logger.error("Build supplies %i archives: %r", len(build_archives), to_list(build_archives.keys())) + logger.error("Build supplies %i archives: %r", + len(build_archives), to_list(build_archives.keys())) if tag_archive: - logger.error("Size mismatch, br: %i, db: %i", fileinfo['size'], tag_archive['size']) - raise koji.BuildrootError('Unknown file in build environment: %s, size: %s' % - ('%s/%s' % (fileinfo['path'], fileinfo['filename']), fileinfo['size'])) + logger.error("Size mismatch, br: %i, db: %i", + fileinfo['size'], tag_archive['size']) + raise koji.BuildrootError( + 'Unknown file in build environment: %s, size: %s' % + ('%s/%s' % (fileinfo['path'], fileinfo['filename']), fileinfo['size'])) return br.updateArchiveList(archives, project) - def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False): + def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None, + with_separate_src=False): """Initialize a new repo for tag""" host = Host() host.verify() - return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event, with_separate_src=with_separate_src) + return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event, + with_separate_src=with_separate_src) def repoDone(self, repo_id, data, expire=False): """Finalize a repo @@ -14175,7 +14379,8 @@ def get_upload_path(reldir, name, create=False, volume=None): if os.path.exists(u_fn): user_id = int(open(u_fn, 'r').read()) if context.session.user_id != user_id: - raise koji.GenericError("Invalid upload directory, not owner: %s" % orig_reldir) + raise koji.GenericError("Invalid upload directory, not owner: %s" % + orig_reldir) else: with open(u_fn, 'w') as fo: fo.write(str(context.session.user_id)) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index cab7edf..8f2c202 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -155,7 +155,9 @@ class HandlerRegistry(object): if x == 0 and func.__code__.co_varnames[x] == "self": continue if func.__defaults__ and func.__code__.co_argcount - x <= len(func.__defaults__): - args.append((func.__code__.co_varnames[x], func.__defaults__[x - func.__code__.co_argcount + len(func.__defaults__)])) + args.append( + (func.__code__.co_varnames[x], + func.__defaults__[x - func.__code__.co_argcount + len(func.__defaults__)])) else: args.append(func.__code__.co_varnames[x]) return args @@ -317,10 +319,11 @@ class ModXMLRPCRequestHandler(object): if self.logger.isEnabledFor(logging.INFO): rusage = resource.getrusage(resource.RUSAGE_SELF) - self.logger.info("Completed method %s for session %s (#%s): %f seconds, rss %s, stime %f", - method, context.session.id, context.session.callnum, - time.time() - start, - rusage.ru_maxrss, rusage.ru_stime) + self.logger.info( + "Completed method %s for session %s (#%s): %f seconds, rss %s, stime %f", + method, context.session.id, context.session.callnum, + time.time() - start, + rusage.ru_maxrss, rusage.ru_stime) return ret @@ -344,8 +347,11 @@ class ModXMLRPCRequestHandler(object): faultCode = getattr(exc_type, 'faultCode', 1) faultString = ', '.join(exc_value.args) trace = traceback.format_exception(*sys.exc_info()) - # traceback is not part of the multicall spec, but we include it for debugging purposes - results.append({'faultCode': faultCode, 'faultString': faultString, 'traceback': trace}) + # traceback is not part of the multicall spec, + # but we include it for debugging purposes + results.append({'faultCode': faultCode, + 'faultString': faultString, + 'traceback': trace}) else: results.append([result]) @@ -438,7 +444,9 @@ def load_config(environ): ['VerbosePolicy', 'boolean', False], ['LogLevel', 'string', 'WARNING'], - ['LogFormat', 'string', '%(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s %(name)s: %(message)s'], + ['LogFormat', 'string', + '%(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s ' + '%(name)s: %(message)s'], ['MissingPolicyOk', 'boolean', True], ['EnableMaven', 'boolean', False], @@ -660,7 +668,8 @@ def load_scripts(environ): def get_memory_usage(): pagesize = resource.getpagesize() - statm = [pagesize * int(y) // 1024 for y in "".join(open("/proc/self/statm").readlines()).strip().split()] + statm = [pagesize * int(y) // 1024 + for y in "".join(open("/proc/self/statm").readlines()).strip().split()] size, res, shr, text, lib, data, dirty = statm return res - shr @@ -713,7 +722,8 @@ def application(environ, start_response): ('Allow', 'POST'), ] start_response('405 Method Not Allowed', headers) - response = "Method Not Allowed\nThis is an XML-RPC server. Only POST requests are accepted." + response = "Method Not Allowed\n" \ + "This is an XML-RPC server. Only POST requests are accepted." if six.PY3: response = response.encode() headers = [ @@ -767,7 +777,11 @@ def application(environ, start_response): paramstr = repr(getattr(context, 'params', 'UNKNOWN')) if len(paramstr) > 120: paramstr = paramstr[:117] + "..." - h.logger.warning("Memory usage of process %d grew from %d KiB to %d KiB (+%d KiB) processing request %s with args %s" % (os.getpid(), memory_usage_at_start, memory_usage_at_end, memory_usage_at_end - memory_usage_at_start, context.method, paramstr)) + h.logger.warning( + "Memory usage of process %d grew from %d KiB to %d KiB (+%d KiB) processing " + "request %s with args %s" % + (os.getpid(), memory_usage_at_start, memory_usage_at_end, + memory_usage_at_end - memory_usage_at_start, context.method, paramstr)) h.logger.debug("Returning %d bytes after %f seconds", len(response), time.time() - start) finally: diff --git a/koji/__init__.py b/koji/__init__.py index cef9af7..82b92c1 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -75,7 +75,7 @@ try: from OpenSSL.SSL import Error as SSL_Error except Exception: # pragma: no cover # the hub imports koji, and sometimes this import fails there - # see: https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine + # see: https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine # noqa: E501 # unfortunately the workaround at the above link does not always work, so # we ignore it here pass @@ -1270,7 +1270,8 @@ def parse_pom(path=None, contents=None): fd.close() if not contents: - raise GenericError('either a path to a pom file or the contents of a pom file must be specified') + raise GenericError( + 'either a path to a pom file or the contents of a pom file must be specified') # A common problem is non-UTF8 characters in XML files, so we'll convert the string first @@ -1287,7 +1288,8 @@ def parse_pom(path=None, contents=None): for field in fields: if field not in util.to_list(values.keys()): - raise GenericError('could not extract %s from POM: %s' % (field, (path or ''))) + raise GenericError('could not extract %s from POM: %s' % + (field, (path or ''))) return values @@ -1649,7 +1651,8 @@ name=build # The following macro values cannot be overridden by tag options macros['%_topdir'] = '%s/build' % config_opts['chroothome'] macros['%_host_cpu'] = opts.get('target_arch', arch) - macros['%_host'] = '%s-%s' % (opts.get('target_arch', arch), opts.get('mockhost', 'koji-linux-gnu')) + macros['%_host'] = '%s-%s' % (opts.get('target_arch', arch), + opts.get('mockhost', 'koji-linux-gnu')) parts = ["""# Auto-generated by the Koji build system """] @@ -1681,7 +1684,9 @@ name=build if bind_opts: for key in bind_opts.keys(): for mnt_src, mnt_dest in six.iteritems(bind_opts.get(key)): - parts.append("config_opts['plugin_conf']['bind_mount_opts'][%r].append((%r, %r))\n" % (key, mnt_src, mnt_dest)) + parts.append( + "config_opts['plugin_conf']['bind_mount_opts'][%r].append((%r, %r))\n" % + (key, mnt_src, mnt_dest)) parts.append("\n") for key in sorted(macros): @@ -1886,7 +1891,8 @@ def read_config(profile_name, user_config=None): try: result[name] = int(value) except ValueError: - raise ConfigurationError("value for %s config option must be a valid integer" % name) + raise ConfigurationError( + "value for %s config option must be a valid integer" % name) else: result[name] = value @@ -2030,7 +2036,8 @@ def read_config_files(config_files, raw=False): class PathInfo(object): # ASCII numbers and upper- and lower-case letter for use in tmpdir() - ASCII_CHARS = [chr(i) for i in list(range(48, 58)) + list(range(65, 91)) + list(range(97, 123))] + ASCII_CHARS = [chr(i) + for i in list(range(48, 58)) + list(range(65, 91)) + list(range(97, 123))] def __init__(self, topdir=None): self._topdir = topdir @@ -2053,10 +2060,12 @@ class PathInfo(object): def build(self, build): """Return the directory where a build belongs""" - return self.volumedir(build.get('volume_name')) + ("/packages/%(name)s/%(version)s/%(release)s" % build) + return self.volumedir(build.get('volume_name')) + \ + ("/packages/%(name)s/%(version)s/%(release)s" % build) def mavenbuild(self, build): - """Return the directory where the Maven build exists in the global store (/mnt/koji/packages)""" + """Return the directory where the Maven build exists in the global store + (/mnt/koji/packages)""" return self.build(build) + '/maven' def mavenrepo(self, maveninfo): @@ -2137,7 +2146,8 @@ class PathInfo(object): """Return a path to a unique directory under work()/tmp/""" tmp = None while tmp is None or os.path.exists(tmp): - tmp = self.work(volume) + '/tmp/' + ''.join([random.choice(self.ASCII_CHARS) for dummy in '123456']) + tmp = self.work(volume) + '/tmp/' + ''.join([random.choice(self.ASCII_CHARS) + for dummy in '123456']) return tmp def scratch(self): @@ -2781,9 +2791,9 @@ class ClientSession(object): # basically, we want to retry on most errors, with a few exceptions # - faults (this means the call completed and failed) # - SystemExit, KeyboardInterrupt - # note that, for logged-in sessions the server should tell us (via a RetryError fault) - # if the call cannot be retried. For non-logged-in sessions, all calls should be read-only - # and hence retryable. + # note that, for logged-in sessions the server should tell us (via a RetryError + # fault) if the call cannot be retried. For non-logged-in sessions, all calls + # should be read-only and hence retryable. except Fault as fault: # try to convert the fault to a known exception err = convertFault(fault) @@ -2792,13 +2802,14 @@ class ClientSession(object): secs = self.opts.get('offline_retry_interval', interval) self.logger.debug("Server offline. Retrying in %i seconds", secs) time.sleep(secs) - # reset try count - this isn't a typical error, this is a running server - # correctly reporting an outage + # reset try count - this isn't a typical error, this is a running + # server correctly reporting an outage tries = 0 continue raise err except (SystemExit, KeyboardInterrupt): - # (depending on the python version, these may or may not be subclasses of Exception) + # (depending on the python version, these may or may not be subclasses of + # Exception) raise except Exception as e: tb_str = ''.join(traceback.format_exception(*sys.exc_info())) @@ -2809,8 +2820,9 @@ class ClientSession(object): raise if not self.logged_in: - # in the past, non-logged-in sessions did not retry. For compatibility purposes - # this behavior is governed by the anon_retry opt. + # in the past, non-logged-in sessions did not retry. + # For compatibility purposes this behavior is governed by the anon_retry + # opt. if not self.opts.get('anon_retry', False): raise @@ -2822,7 +2834,8 @@ class ClientSession(object): # otherwise keep retrying if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug(tb_str) - self.logger.info("Try #%s for call %s (%s) failed: %s", tries, self.callnum, name, e) + self.logger.info("Try #%s for call %s (%s) failed: %s", + tries, self.callnum, name, e) if tries > 1: # first retry is immediate, after that we honor retry_interval time.sleep(interval) @@ -2864,7 +2877,8 @@ class ClientSession(object): transaction. """ if not self.multicall: - raise GenericError('ClientSession.multicall must be set to True before calling multiCall()') + raise GenericError( + 'ClientSession.multicall must be set to True before calling multiCall()') self.multicall = False if len(self._calls) == 0: return [] @@ -2896,7 +2910,8 @@ class ClientSession(object): return self.__dict__['_apidoc'] return VirtualMethod(self._callMethod, name, self) - def fastUpload(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=False, volume=None): + def fastUpload(self, localfile, path, name=None, callback=None, blocksize=None, + overwrite=False, volume=None): if blocksize is None: blocksize = self.opts.get('upload_blocksize', 1048576) @@ -2930,7 +2945,8 @@ class ClientSession(object): hexdigest = util.adler32_constructor(chunk).hexdigest() full_chksum.update(chunk) if result['size'] != len(chunk): - raise GenericError("server returned wrong chunk size: %s != %s" % (result['size'], len(chunk))) + raise GenericError("server returned wrong chunk size: %s != %s" % + (result['size'], len(chunk))) if result['hexdigest'] != hexdigest: raise GenericError('upload checksum failed: %s != %s' % (result['hexdigest'], hexdigest)) @@ -2957,9 +2973,11 @@ class ClientSession(object): if problems and result['hexdigest'] != full_chksum.hexdigest(): raise GenericError("Uploaded file has wrong checksum: %s/%s, %s != %s" % (path, name, result['hexdigest'], full_chksum.hexdigest())) - self.logger.debug("Fast upload: %s complete. %i bytes in %.1f seconds", localfile, size, t2) + self.logger.debug("Fast upload: %s complete. %i bytes in %.1f seconds", + localfile, size, t2) - def _prepUpload(self, chunk, offset, path, name, verify="adler32", overwrite=False, volume=None): + def _prepUpload(self, chunk, offset, path, name, verify="adler32", overwrite=False, + volume=None): """prep a rawUpload call""" if not self.logged_in: raise ActionNotAllowed("you must be logged in to upload") @@ -2989,7 +3007,8 @@ class ClientSession(object): request = chunk return handler, headers, request - def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=True, volume=None): + def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=None, + overwrite=True, volume=None): """upload a file in chunks using the uploadFile call""" if blocksize is None: blocksize = self.opts.get('upload_blocksize', 1048576) @@ -3044,7 +3063,8 @@ class ClientSession(object): tries = 0 while True: if debug: - self.logger.debug("uploadFile(%r,%r,%r,%r,%r,...)" % (path, name, sz, digest, offset)) + self.logger.debug("uploadFile(%r,%r,%r,%r,%r,...)" % + (path, name, sz, digest, offset)) if self.callMethod('uploadFile', path, name, sz, digest, offset, data, **volopts): break if tries <= retries: @@ -3063,9 +3083,11 @@ class ClientSession(object): if t2 <= 0: t2 = 1 if debug: - self.logger.debug("Uploaded %d bytes in %f seconds (%f kbytes/sec)" % (size, t1, size / t1 / 1024.0)) + self.logger.debug("Uploaded %d bytes in %f seconds (%f kbytes/sec)" % + (size, t1, size / t1 / 1024.0)) if debug: - self.logger.debug("Total: %d bytes in %f seconds (%f kbytes/sec)" % (ofs, t2, ofs / t2 / 1024.0)) + self.logger.debug("Total: %d bytes in %f seconds (%f kbytes/sec)" % + (ofs, t2, ofs / t2 / 1024.0)) if callback: callback(ofs, totalsize, size, t1, t2) fo.close() @@ -3281,8 +3303,8 @@ class DBHandler(logging.Handler): cursor.execute(command, data) cursor.close() # self.cnx.commit() - # XXX - committing here is most likely wrong, but we need to set commit_pending or something - # ...and this is really the wrong place for that + # XXX - committing here is most likely wrong, but we need to set commit_pending or + # something...and this is really the wrong place for that except BaseException: self.handleError(record) @@ -3583,7 +3605,9 @@ def add_file_logger(logger, fn): def add_stderr_logger(logger): handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] {%(process)d} %(name)s:%(lineno)d %(message)s')) + handler.setFormatter( + logging.Formatter( + '%(asctime)s [%(levelname)s] {%(process)d} %(name)s:%(lineno)d %(message)s')) handler.setLevel(logging.DEBUG) logging.getLogger(logger).addHandler(handler) @@ -3612,7 +3636,8 @@ def add_mail_logger(logger, addr): return addresses = addr.split(',') handler = logging.handlers.SMTPHandler("localhost", - "%s@%s" % (pwd.getpwuid(os.getuid())[0], socket.getfqdn()), + "%s@%s" % (pwd.getpwuid(os.getuid())[0], + socket.getfqdn()), addresses, "%s: error notice" % socket.getfqdn()) handler.setFormatter(logging.Formatter('%(pathname)s:%(lineno)d [%(levelname)s] %(message)s')) diff --git a/koji/auth.py b/koji/auth.py index 3a74a7c..1a3eef6 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -334,7 +334,8 @@ class Session(object): # Successfully authenticated via Kerberos, now log in if proxyuser: - proxyprincs = [princ.strip() for princ in context.opts.get('ProxyPrincipals', '').split(',')] + proxyprincs = [princ.strip() + for princ in context.opts.get('ProxyPrincipals', '').split(',')] if cprinc.name in proxyprincs: login_principal = proxyuser else: @@ -408,12 +409,15 @@ class Session(object): authtype = koji.AUTHTYPE_GSSAPI else: if context.environ.get('SSL_CLIENT_VERIFY') != 'SUCCESS': - raise koji.AuthError('could not verify client: %s' % context.environ.get('SSL_CLIENT_VERIFY')) + raise koji.AuthError('could not verify client: %s' % + context.environ.get('SSL_CLIENT_VERIFY')) name_dn_component = context.opts.get('DNUsernameComponent', 'CN') username = context.environ.get('SSL_CLIENT_S_DN_%s' % name_dn_component) if not username: - raise koji.AuthError('unable to get user information (%s) from client certificate' % name_dn_component) + raise koji.AuthError( + 'unable to get user information (%s) from client certificate' % + name_dn_component) client_dn = context.environ.get('SSL_CLIENT_S_DN') authtype = koji.AUTHTYPE_SSL diff --git a/koji/daemon.py b/koji/daemon.py index c3adad7..ad425d9 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -110,8 +110,9 @@ def fast_incremental_upload(session, fname, fd, path, retries, logger): break -def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, append=0, chroot=None, env=None): - """Run command with output redirected. If chroot is not None, chroot to the directory specified +def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, append=0, + chroot=None, env=None): + """Run command with output redirected. If chroot is not None, chroot to the directory specified before running the command.""" pid = os.fork() fd = None @@ -287,11 +288,13 @@ class SCM(object): elif len(userhost) > 2: raise koji.GenericError('Invalid username@hostname specified: %s' % netloc) if not netloc: - raise koji.GenericError('Unable to parse SCM URL: %s . Could not find the netloc element.' % self.url) + raise koji.GenericError( + 'Unable to parse SCM URL: %s . Could not find the netloc element.' % self.url) # check for empty path before we apply normpath if not path: - raise koji.GenericError('Unable to parse SCM URL: %s . Could not find the path element.' % self.url) + raise koji.GenericError( + 'Unable to parse SCM URL: %s . Could not find the path element.' % self.url) path = os.path.normpath(path) @@ -306,14 +309,19 @@ class SCM(object): # any such url should have already been caught by is_scm_url raise koji.GenericError('Invalid SCM URL. Path should begin with /: %s) ') - # check for validity: params should be empty, query may be empty, everything else should be populated + # check for validity: params should be empty, query may be empty, everything else should be + # populated if params: - raise koji.GenericError('Unable to parse SCM URL: %s . Params element %s should be empty.' % (self.url, params)) + raise koji.GenericError( + 'Unable to parse SCM URL: %s . Params element %s should be empty.' % + (self.url, params)) if not scheme: # pragma: no cover # should not happen because of is_scm_url check earlier - raise koji.GenericError('Unable to parse SCM URL: %s . Could not find the scheme element.' % self.url) + raise koji.GenericError( + 'Unable to parse SCM URL: %s . Could not find the scheme element.' % self.url) if not fragment: - raise koji.GenericError('Unable to parse SCM URL: %s . Could not find the fragment element.' % self.url) + raise koji.GenericError( + 'Unable to parse SCM URL: %s . Could not find the fragment element.' % self.url) # return parsed values return (scheme, user, netloc, path, query, fragment) @@ -356,7 +364,8 @@ class SCM(object): for allowed_scm in allowed.split(): scm_tuple = allowed_scm.split(':') if len(scm_tuple) < 2: - self.logger.warn('Ignoring incorrectly formatted SCM host:repository: %s' % allowed_scm) + self.logger.warn('Ignoring incorrectly formatted SCM host:repository: %s' % + allowed_scm) continue host_pat = scm_tuple[0] repo_pat = scm_tuple[1] @@ -378,11 +387,13 @@ class SCM(object): if scm_tuple[3]: self.source_cmd = scm_tuple[3].split(',') else: - # there was nothing after the trailing :, so they don't want to run a source_cmd at all + # there was nothing after the trailing :, + # so they don't want to run a source_cmd at all self.source_cmd = None break if not is_allowed: - raise koji.BuildError('%s:%s is not in the list of allowed SCMs' % (self.host, self.repository)) + raise koji.BuildError( + '%s:%s is not in the list of allowed SCMs' % (self.host, self.repository)) def checkout(self, scmdir, session=None, uploadpath=None, logfile=None): """ @@ -416,16 +427,20 @@ class SCM(object): (self.scmtype, ' '.join(cmd), os.path.basename(logfile))) if self.scmtype == 'CVS': - pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, self.repository) - module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, self.module] + pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, + self.repository) + module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, + self.module] common_checkout_cmd = ['cvs', '-d', pserver, 'checkout', 'common'] elif self.scmtype == 'CVS+SSH': if not self.user: - raise koji.BuildError('No user specified for repository access scheme: %s' % self.scheme) + raise koji.BuildError( + 'No user specified for repository access scheme: %s' % self.scheme) cvsserver = ':ext:%s@%s:%s' % (self.user, self.host, self.repository) - module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, self.module] + module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, + self.module] common_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', 'common'] env = {'CVS_RSH': 'ssh'} @@ -453,14 +468,16 @@ class SCM(object): update_checkout_cmd = ['git', 'reset', '--hard', self.revision] update_checkout_dir = sourcedir - # self.module may be empty, in which case the specfile should be in the top-level directory + # self.module may be empty, in which case the specfile should be in the top-level + # directory if self.module: # Treat the module as a directory inside the git repository sourcedir = '%s/%s' % (sourcedir, self.module) elif self.scmtype == 'GIT+SSH': if not self.user: - raise koji.BuildError('No user specified for repository access scheme: %s' % self.scheme) + raise koji.BuildError( + 'No user specified for repository access scheme: %s' % self.scheme) gitrepo = 'git+ssh://%s@%s%s' % (self.user, self.host, self.repository) commonrepo = os.path.dirname(gitrepo) + '/common' checkout_path = os.path.basename(self.repository) @@ -481,7 +498,8 @@ class SCM(object): update_checkout_cmd = ['git', 'reset', '--hard', self.revision] update_checkout_dir = sourcedir - # self.module may be empty, in which case the specfile should be in the top-level directory + # self.module may be empty, in which case the specfile should be in the top-level + # directory if self.module: # Treat the module as a directory inside the git repository sourcedir = '%s/%s' % (sourcedir, self.module) @@ -492,15 +510,18 @@ class SCM(object): scheme = scheme.split('+')[1] svnserver = '%s%s%s' % (scheme, self.host, self.repository) - module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module] + module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, + '%s/%s' % (svnserver, self.module), self.module] common_checkout_cmd = ['svn', 'checkout', '%s/common' % svnserver] elif self.scmtype == 'SVN+SSH': if not self.user: - raise koji.BuildError('No user specified for repository access scheme: %s' % self.scheme) + raise koji.BuildError( + 'No user specified for repository access scheme: %s' % self.scheme) svnserver = 'svn+ssh://%s@%s%s' % (self.user, self.host, self.repository) - module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module] + module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, + '%s/%s' % (svnserver, self.module), self.module] common_checkout_cmd = ['svn', 'checkout', '%s/common' % svnserver] else: @@ -513,8 +534,10 @@ class SCM(object): # Currently only required for GIT checkouts # Run the command in the directory the source was checked out into if self.scmtype.startswith('GIT') and globals().get('KOJIKAMID'): - _run(['git', 'config', 'core.autocrlf', 'true'], chdir=update_checkout_dir, fatal=True) - _run(['git', 'config', 'core.safecrlf', 'true'], chdir=update_checkout_dir, fatal=True) + _run(['git', 'config', 'core.autocrlf', 'true'], + chdir=update_checkout_dir, fatal=True) + _run(['git', 'config', 'core.safecrlf', 'true'], + chdir=update_checkout_dir, fatal=True) _run(update_checkout_cmd, chdir=update_checkout_dir, fatal=True) if self.use_common and not globals().get('KOJIKAMID'): @@ -583,7 +606,8 @@ class TaskManager(object): def registerHandler(self, entry): """register and index task handler""" - if isinstance(entry, type(koji.tasks.BaseTaskHandler)) and issubclass(entry, koji.tasks.BaseTaskHandler): + if isinstance(entry, type(koji.tasks.BaseTaskHandler)) and \ + issubclass(entry, koji.tasks.BaseTaskHandler): for method in entry.Methods: self.handlers[method] = entry @@ -638,7 +662,9 @@ class TaskManager(object): # task not running - expire the buildroot # TODO - consider recycling hooks here (with strong sanity checks) self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br) - self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, to_list(self.tasks.keys()))) + self.logger.debug( + "Buildroot task: %r, Current tasks: %r" % + (task_id, to_list(self.tasks.keys()))) self.session.host.setBuildRootState(id, st_expired) continue if nolocal: @@ -678,7 +704,8 @@ class TaskManager(object): if not task: self.logger.warn("%s: invalid task %s" % (desc, br['task_id'])) continue - if (task['state'] == koji.TASK_STATES['FAILED'] and age < self.options.failed_buildroot_lifetime): + if task['state'] == koji.TASK_STATES['FAILED'] and \ + age < self.options.failed_buildroot_lifetime: # XXX - this could be smarter # keep buildroots for failed tasks around for a little while self.logger.debug("Keeping failed buildroot: %s" % desc) @@ -1004,7 +1031,9 @@ class TaskManager(object): self.logger.info('%s (pid %i, taskID %i) is running' % (execname, pid, task_id)) else: if signaled: - self.logger.info('%s (pid %i, taskID %i) was killed by signal %i' % (execname, pid, task_id, sig)) + self.logger.info( + '%s (pid %i, taskID %i) was killed by signal %i' % + (execname, pid, task_id, sig)) else: self.logger.info('%s (pid %i, taskID %i) exited' % (execname, pid, task_id)) return True @@ -1041,7 +1070,8 @@ class TaskManager(object): if not os.path.isfile(proc_path): return None proc_file = open(proc_path) - procstats = [not field.isdigit() and field or int(field) for field in proc_file.read().split()] + procstats = [not field.isdigit() and field or int(field) + for field in proc_file.read().split()] proc_file.close() cmd_path = '/proc/%i/cmdline' % pid @@ -1084,9 +1114,9 @@ class TaskManager(object): while parents: for ppid in parents[:]: for procstats in statsByPPID.get(ppid, []): - # get the /proc entries with ppid as their parent, and append their pid to the list, - # then recheck for their children - # pid is the 0th field, ppid is the 3rd field + # get the /proc entries with ppid as their parent, and append their pid to the + # list, then recheck for their children pid is the 0th field, ppid is the 3rd + # field pids.append((procstats[0], procstats[1])) parents.append(procstats[0]) parents.remove(ppid) @@ -1154,7 +1184,8 @@ class TaskManager(object): availableMB = available // 1024 // 1024 self.logger.debug("disk space available in '%s': %i MB", br_path, availableMB) if availableMB < self.options.minspace: - self.status = "Insufficient disk space at %s: %i MB, %i MB required" % (br_path, availableMB, self.options.minspace) + self.status = "Insufficient disk space at %s: %i MB, %i MB required" % \ + (br_path, availableMB, self.options.minspace) self.logger.warn(self.status) return False return True @@ -1189,7 +1220,9 @@ class TaskManager(object): return False if self.task_load > self.hostdata['capacity']: self.status = "Over capacity" - self.logger.info("Task load (%.2f) exceeds capacity (%.2f)" % (self.task_load, self.hostdata['capacity'])) + self.logger.info( + "Task load (%.2f) exceeds capacity (%.2f)" % + (self.task_load, self.hostdata['capacity'])) return False if len(self.tasks) >= self.options.maxjobs: # This serves as a backup to the capacity check and prevents @@ -1238,7 +1271,8 @@ class TaskManager(object): self.logger.warn('Error during host check') self.logger.warn(''.join(traceback.format_exception(*sys.exc_info()))) if not valid_host: - self.logger.info('Skipping task %s (%s) due to host check', task['id'], task['method']) + self.logger.info( + 'Skipping task %s (%s) due to host check', task['id'], task['method']) return False data = self.session.host.openTask(task['id']) if data is None: diff --git a/koji/db.py b/koji/db.py index f7911a9..29042ad 100644 --- a/koji/db.py +++ b/koji/db.py @@ -110,7 +110,8 @@ class CursorWrapper: try: return quote(operation, parameters) except Exception: - self.logger.exception('Unable to quote query:\n%s\nParameters: %s', operation, parameters) + self.logger.exception( + 'Unable to quote query:\n%s\nParameters: %s', operation, parameters) return "INVALID QUERY" def preformat(self, sql, params): diff --git a/koji/tasks.py b/koji/tasks.py index 440eaa0..7ceac2f 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -154,10 +154,14 @@ LEGACY_SIGNATURES = { [['tag', 'newer_than', 'nvrs'], None, None, (None, None)], ], 'createLiveMedia': [ - [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'ksfile', 'opts'], None, None, (None,)], + [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'ksfile', + 'opts'], + None, None, (None,)], ], 'createAppliance': [ - [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'ksfile', 'opts'], None, None, (None,)], + [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'ksfile', + 'opts'], + None, None, (None,)], ], 'livecd': [ [['name', 'version', 'arch', 'target', 'ksfile', 'opts'], None, None, (None,)], @@ -190,7 +194,9 @@ LEGACY_SIGNATURES = { [['spec_url', 'build_target', 'build', 'task', 'opts'], None, None, (None,)], ], 'createLiveCD': [ - [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'ksfile', 'opts'], None, None, (None,)], + [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'ksfile', + 'opts'], + None, None, (None,)], ], 'appliance': [ [['name', 'version', 'arch', 'target', 'ksfile', 'opts'], None, None, (None,)], @@ -199,19 +205,25 @@ LEGACY_SIGNATURES = { [['name', 'version', 'arches', 'target', 'inst_tree', 'opts'], None, None, (None,)], ], 'tagBuild': [ - [['tag_id', 'build_id', 'force', 'fromtag', 'ignore_success'], None, None, (False, None, False)], + [['tag_id', 'build_id', 'force', 'fromtag', 'ignore_success'], + None, None, (False, None, False)], ], 'chainmaven': [ [['builds', 'target', 'opts'], None, None, (None,)], ], 'newRepo': [ - [['tag', 'event', 'src', 'debuginfo', 'separate_src'], None, None, (None, False, False, False)], + [['tag', 'event', 'src', 'debuginfo', 'separate_src'], + None, None, (None, False, False, False)], ], 'createImage': [ - [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'inst_tree', 'opts'], None, None, (None,)], + [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', + 'inst_tree', 'opts'], + None, None, (None,)], ], 'tagNotification': [ - [['recipients', 'is_successful', 'tag_info', 'from_info', 'build_info', 'user_info', 'ignore_success', 'failure_msg'], None, None, (None, '')], + [['recipients', 'is_successful', 'tag_info', 'from_info', 'build_info', 'user_info', + 'ignore_success', 'failure_msg'], + None, None, (None, '')], ], 'buildArch': [ [['pkg', 'root', 'arch', 'keep_srpm', 'opts'], None, None, (None,)], @@ -253,7 +265,9 @@ LEGACY_SIGNATURES = { [['options'], None, None, (None,)], ], 'runroot': [ - [['root', 'arch', 'command', 'keep', 'packages', 'mounts', 'repo_id', 'skip_setarch', 'weight', 'upload_logs', 'new_chroot'], None, None, (False, [], [], None, False, None, None, False)], + [['root', 'arch', 'command', 'keep', 'packages', 'mounts', 'repo_id', 'skip_setarch', + 'weight', 'upload_logs', 'new_chroot'], + None, None, (False, [], [], None, False, None, None, False)], ], 'distRepo': [ [['tag', 'repo_id', 'keys', 'task_opts'], None, None, None], @@ -400,7 +414,9 @@ class BaseTaskHandler(object): self.session.getTaskResult(task) checked.add(task) except (koji.GenericError, six.moves.xmlrpc_client.Fault): - self.logger.info("task %s failed or was canceled, cancelling unfinished tasks" % task) + self.logger.info( + "task %s failed or was canceled, cancelling unfinished tasks" % + task) self.session.cancelTaskChildren(self.id) # reraise the original error now, rather than waiting for # an error in taskWaitResults() @@ -743,8 +759,10 @@ class RestartHostsTask(BaseTaskHandler): my_tasks = None for host in hosts: # note: currently task assignments bypass channel restrictions - task1 = self.subtask('restart', [host], assign=host['id'], label="restart %i" % host['id']) - task2 = self.subtask('restartVerify', [task1, host], assign=host['id'], label="sleep %i" % host['id']) + task1 = self.subtask('restart', [host], + assign=host['id'], label="restart %i" % host['id']) + task2 = self.subtask('restartVerify', [task1, host], + assign=host['id'], label="sleep %i" % host['id']) subtasks.append(task1) subtasks.append(task2) if host['id'] == this_host: @@ -790,8 +808,10 @@ class DependantTask(BaseTaskHandler): subtasks = [] for task in task_list: - # **((len(task)>2 and task[2]) or {}) expands task[2] into opts if it exists, allows for things like 'priority=15' - task_id = self.session.host.subtask(method=task[0], arglist=task[1], parent=self.id, **((len(task) > 2 and task[2]) or {})) + # **((len(task)>2 and task[2]) or {}) expands task[2] into opts if it exists, allows + # for things like 'priority=15' + task_id = self.session.host.subtask(method=task[0], arglist=task[1], parent=self.id, + **((len(task) > 2 and task[2]) or {})) if task_id: subtasks.append(task_id) if subtasks: diff --git a/koji/util.py b/koji/util.py index 919f592..745362d 100644 --- a/koji/util.py +++ b/koji/util.py @@ -54,7 +54,8 @@ def deprecated(message): def _changelogDate(cldate): - return time.strftime('%a %b %d %Y', time.strptime(koji.formatTime(cldate), '%Y-%m-%d %H:%M:%S')) + return time.strftime('%a %b %d %Y', + time.strptime(koji.formatTime(cldate), '%Y-%m-%d %H:%M:%S')) def formatChangelog(entries): @@ -813,7 +814,8 @@ def parse_maven_param(confs, chain=False, scratch=False, section=None): else: raise ValueError("Section %s does not exist in: %s" % (section, ', '.join(confs))) elif len(builds) > 1: - raise ValueError("Multiple sections in: %s, you must specify the section" % ', '.join(confs)) + raise ValueError( + "Multiple sections in: %s, you must specify the section" % ', '.join(confs)) return builds diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index 31552c0..d267192 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -47,7 +47,8 @@ class RunRootTask(koji.tasks.BaseTaskHandler): options.append(o) rel_path = path[len(mount_data['mountpoint']):] rel_path = rel_path[1:] if rel_path.startswith('/') else rel_path - res = (os.path.join(mount_data['path'], rel_path), path, mount_data['fstype'], ','.join(options)) + res = (os.path.join(mount_data['path'], rel_path), path, mount_data['fstype'], + ','.join(options)) return res def _read_config(self): @@ -94,11 +95,15 @@ class RunRootTask(koji.tasks.BaseTaskHandler): except six.moves.configparser.NoOptionError: raise koji.GenericError("bad config: missing options in %s section" % section_name) - for path in self.config['default_mounts'] + self.config['safe_roots'] + [x[0] for x in self.config['path_subs']]: + for path in self.config['default_mounts'] + self.config['safe_roots'] + \ + [x[0] for x in self.config['path_subs']]: if not path.startswith('/'): - raise koji.GenericError("bad config: all paths (default_mounts, safe_roots, path_subs) needs to be absolute: %s" % path) + raise koji.GenericError( + "bad config: all paths (default_mounts, safe_roots, path_subs) needs to be " + "absolute: %s" % path) - def handler(self, root, arch, command, keep=False, packages=[], mounts=[], repo_id=None, skip_setarch=False, weight=None, upload_logs=None, new_chroot=None): + def handler(self, root, arch, command, keep=False, packages=[], mounts=[], repo_id=None, + skip_setarch=False, weight=None, upload_logs=None, new_chroot=None): """Create a buildroot and run a command (as root) inside of it Command may be a string or a list. @@ -141,15 +146,19 @@ class RunRootTask(koji.tasks.BaseTaskHandler): break else: # no overlap - raise koji.BuildError("host does not match tag arches: %s (%s)" % (root, tag_arches)) + raise koji.BuildError( + "host does not match tag arches: %s (%s)" % (root, tag_arches)) else: br_arch = arch if repo_id: repo_info = self.session.repoInfo(repo_id, strict=True) if repo_info['tag_name'] != root: - raise koji.BuildError("build tag (%s) does not match repo tag (%s)" % (root, repo_info['tag_name'])) + raise koji.BuildError( + "build tag (%s) does not match repo tag (%s)" % (root, repo_info['tag_name'])) if repo_info['state'] not in (koji.REPO_STATES['READY'], koji.REPO_STATES['EXPIRED']): - raise koji.BuildError("repos in the %s state may not be used by runroot" % koji.REPO_STATES[repo_info['state']]) + raise koji.BuildError( + "repos in the %s state may not be used by runroot" % + koji.REPO_STATES[repo_info['state']]) else: repo_info = self.session.getRepo(root) if not repo_info: @@ -186,12 +195,15 @@ class RunRootTask(koji.tasks.BaseTaskHandler): cmdstr = ' '.join(["'%s'" % arg.replace("'", r"'\''") for arg in command]) # A nasty hack to put command output into its own file until mock can be # patched to do something more reasonable than stuff everything into build.log - cmdargs = ['/bin/sh', '-c', "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /builddir/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr] + cmdargs = ['/bin/sh', '-c', + "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /builddir/runroot.log; exit " + "${PIPESTATUS[0]}" % cmdstr] # always mount /mnt/redhat (read-only) # always mount /mnt/iso (read-only) # also need /dev bind mount - self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config['default_mounts']]) + self.do_mounts(rootdir, + [self._get_path_params(x) for x in self.config['default_mounts']]) self.do_extra_mounts(rootdir, mounts) mock_cmd = ['chroot'] if new_chroot: @@ -199,7 +211,8 @@ class RunRootTask(koji.tasks.BaseTaskHandler): elif new_chroot is False: # None -> no option added mock_cmd.append('--old-chroot') if skip_setarch: - # we can't really skip it, but we can set it to the current one instead of of the chroot one + # we can't really skip it, but we can set it to the current one instead of of the + # chroot one myarch = platform.uname()[5] mock_cmd.extend(['--arch', myarch]) mock_cmd.append('--') @@ -279,7 +292,8 @@ class RunRootTask(koji.tasks.BaseTaskHandler): cmd = ['mount', '-t', type, '-o', opts, dev, mpoint] self.logger.info("Mount command: %r" % cmd) koji.ensuredir(mpoint) - status = log_output(self.session, cmd[0], cmd, logfile, uploadpath, logerror=True, append=True) + status = log_output(self.session, cmd[0], cmd, logfile, uploadpath, + logerror=True, append=True) if not isSuccess(status): error = koji.GenericError("Unable to mount %s: %s" % (mpoint, parseStatus(status, cmd))) @@ -306,7 +320,8 @@ class RunRootTask(koji.tasks.BaseTaskHandler): failed = [] self.logger.info("Unmounting (runroot): %s" % mounts) for dir in mounts: - proc = subprocess.Popen(["umount", "-l", dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen(["umount", "-l", dir], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.wait() != 0: output = proc.stdout.read() output += proc.stderr.read() diff --git a/plugins/cli/runroot.py b/plugins/cli/runroot.py index 7c9a16a..f31e950 100644 --- a/plugins/cli/runroot.py +++ b/plugins/cli/runroot.py @@ -22,8 +22,10 @@ def handle_runroot(options, session, args): usage += _("\n(Specify the --help global option for a list of other help options)") parser = OptionParser(usage=usage) parser.disable_interspersed_args() - parser.add_option("-p", "--package", action="append", default=[], help=_("make sure this package is in the chroot")) - parser.add_option("-m", "--mount", action="append", default=[], help=_("mount this directory read-write in the chroot")) + parser.add_option("-p", "--package", action="append", default=[], + help=_("make sure this package is in the chroot")) + parser.add_option("-m", "--mount", action="append", default=[], + help=_("mount this directory read-write in the chroot")) parser.add_option("--skip-setarch", action="store_true", default=False, help=_("bypass normal setarch in the chroot")) parser.add_option("-w", "--weight", type='int', help=_("set task weight")) @@ -39,7 +41,8 @@ def handle_runroot(options, session, args): parser.add_option("--repo-id", type="int", help=_("ID of the repo to use")) parser.add_option("--nowait", action="store_false", dest="wait", default=True, help=_("Do not wait on task")) - parser.add_option("--watch", action="store_true", help=_("Watch task instead of printing runroot.log")) + parser.add_option("--watch", action="store_true", + help=_("Watch task instead of printing runroot.log")) parser.add_option("--quiet", action="store_true", default=options.quiet, help=_("Do not print the task information")) diff --git a/plugins/cli/save_failed_tree.py b/plugins/cli/save_failed_tree.py index 7bfee70..91dca02 100644 --- a/plugins/cli/save_failed_tree.py +++ b/plugins/cli/save_failed_tree.py @@ -14,7 +14,8 @@ def handle_save_failed_tree(options, session, args): usage += _("\n(Specify the --help global option for a list of other help options)") parser = OptionParser(usage=usage) parser.add_option("-f", "--full", action="store_true", default=False, - help=_("Download whole tree, if not specified, only builddir will be downloaded")) + help=_("Download whole tree, if not specified, " + "only builddir will be downloaded")) parser.add_option("-t", "--task", action="store_const", dest="mode", const="task", default="task", help=_("Treat ID as a task ID (the default)")) @@ -69,4 +70,5 @@ def handle_save_failed_tree(options, session, args): return else: session.logout() - return watch_tasks(session, [task_id], quiet=opts.quiet, poll_interval=options.poll_interval) + return watch_tasks(session, [task_id], + quiet=opts.quiet, poll_interval=options.poll_interval) diff --git a/plugins/hub/save_failed_tree.py b/plugins/hub/save_failed_tree.py index beb18e6..0079223 100644 --- a/plugins/hub/save_failed_tree.py +++ b/plugins/hub/save_failed_tree.py @@ -40,10 +40,12 @@ def saveFailedTree(buildrootID, full=False, **opts): taskID = brinfo['task_id'] task_info = kojihub.Task(taskID).getInfo() if task_info['state'] != koji.TASK_STATES['FAILED']: - raise koji.PreBuildError("Task %s has not failed. Only failed tasks can upload their buildroots." % taskID) + raise koji.PreBuildError( + "Task %s has not failed. Only failed tasks can upload their buildroots." % taskID) elif allowed_methods != '*' and task_info['method'] not in allowed_methods: - raise koji.PreBuildError("Only %s tasks can upload their buildroots (Task %s is %s)." % - (', '.join(allowed_methods), task_info['id'], task_info['method'])) + raise koji.PreBuildError( + "Only %s tasks can upload their buildroots (Task %s is %s)." % + (', '.join(allowed_methods), task_info['id'], task_info['method'])) elif task_info["owner"] != context.session.user_id and not context.session.hasPerm('admin'): raise koji.ActionNotAllowed("Only owner of failed task or 'admin' can run this task.") elif not kojihub.get_host(task_info['host_id'])['enabled']: diff --git a/tests/test_cli/test_list_tagged.py b/tests/test_cli/test_list_tagged.py index dd20fdc..0bf81b8 100644 --- a/tests/test_cli/test_list_tagged.py +++ b/tests/test_cli/test_list_tagged.py @@ -251,7 +251,7 @@ Options: --quiet Do not print the header information --paths Show the file paths --sigs Show signatures - --type=TYPE Show builds of the given type only. Currently supported + --type=TYPE Show builds of the given type only. Currently supported types: maven, win, image --event=EVENT# query at event --ts=TIMESTAMP query at last event before timestamp diff --git a/util/koji-gc b/util/koji-gc index ea9fa89..74fd0b8 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -364,7 +364,8 @@ def ensure_connection(session): except requests.exceptions.ConnectionError: error(_("Error: Unable to connect to server")) if ret != koji.API_VERSION: - warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) + warn(_("WARNING: The server is at API version %d and the client is at %d" % + (ret, koji.API_VERSION))) def has_krb_creds(): @@ -394,7 +395,8 @@ def activate_session(session): elif has_krb_creds() or (options.keytab and options.principal): try: if options.keytab and options.principal: - session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) + session.krb_login(principal=options.principal, keytab=options.keytab, + proxyuser=options.runas) else: session.krb_login(proxyuser=options.runas) except krbV.Krb5Error as e: @@ -503,7 +505,8 @@ def handle_trash(): continue if refs.get('archives'): if options.debug: - print("[%i/%i] Build has %i archive references: %s" % (i, N, len(refs['archives']), nvr)) + print("[%i/%i] Build has %i archive references: %s" % + (i, N, len(refs['archives']), nvr)) # pprint.pprint(refs['archives']) continue if refs.get('component_of'): @@ -941,7 +944,8 @@ def handle_prune(): else: print("Untagging build %s from %s" % (nvr, tagname)) try: - session.untagBuildBypass(taginfo['id'], entry['build_id'], force=bypass) + session.untagBuildBypass(taginfo['id'], entry['build_id'], + force=bypass) untagged.setdefault(nvr, {})[tagname] = 1 except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e: print("Warning: untag operation failed: %s" % e) diff --git a/util/koji-shadow b/util/koji-shadow index e388551..55490e4 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -145,13 +145,15 @@ def get_options(): parser.add_option("--rules-ignorelist", help=_("Rules: list of packages to ignore")) parser.add_option("--rules-excludelist", - help=_("Rules: list of packages to are excluded using ExcludeArch or ExclusiveArch")) + help=_("Rules: list of packages to are excluded using ExcludeArch or " + "ExclusiveArch")) parser.add_option("--rules-includelist", help=_("Rules: list of packages to always include")) parser.add_option("--rules-protectlist", help=_("Rules: list of package names to never replace")) parser.add_option("--tag-build", action="store_true", default=False, - help=_("tag successful builds into the tag we are building, default is to not tag")) + help=_("tag successful builds into the tag we are building, default is to " + "not tag")) parser.add_option("--logfile", help=_("file where everything gets logged")) parser.add_option("--arches", @@ -298,14 +300,16 @@ def activate_session(session): if os.path.isfile(options.auth_cert): # authenticate using SSL client cert - session.ssl_login(cert=options.auth_cert, serverca=options.serverca, proxyuser=options.runas) + session.ssl_login(cert=options.auth_cert, serverca=options.serverca, + proxyuser=options.runas) elif options.user: # authenticate using user/password session.login() elif krbV: try: if options.keytab and options.principal: - session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) + session.krb_login(principal=options.principal, keytab=options.keytab, + proxyuser=options.runas) else: session.krb_login(proxyuser=options.runas) except krbV.Krb5Error as e: @@ -537,12 +541,14 @@ class TrackedBuild(object): # each buildroot had this as a base package base.append(name) if len(tags) > 1: - log("Warning: found multiple buildroot tags for %s: %s" % (self.nvr, to_list(tags.keys()))) + log("Warning: found multiple buildroot tags for %s: %s" % + (self.nvr, to_list(tags.keys()))) counts = sorted([(n, tag) for tag, n in six.iteritems(tags)]) tag = counts[-1][1] else: tag = to_list(tags.keys())[0] - # due bugs in used tools mainline koji instance could store empty buildroot infos for builds + # due bugs in used tools mainline koji instance could store empty buildroot infos for + # builds if len(builds) == 0: self.setState("noroot") self.deps = builds @@ -655,7 +661,8 @@ class BuildTracker(object): return -1 def newerBuild(self, build, tag): - # XXX: secondary arches need a policy to say if we have newer build localy it will be the substitute + # XXX: secondary arches need a policy to say if we have newer build localy it will be the + # substitute localBuilds = session.listTagged(tag, inherit=True, package=str(build.name)) newer = None parentevr = (str(build.epoch), build.version, build.release) @@ -664,14 +671,16 @@ class BuildTracker(object): latestevr = (str(b['epoch']), b['version'], b['release']) newestRPM = self.rpmvercmp(parentevr, latestevr) if options.debug: - log("remote evr: %s \nlocal evr: %s \nResult: %s" % (parentevr, latestevr, newestRPM)) + log("remote evr: %s \nlocal evr: %s \nResult: %s" % + (parentevr, latestevr, newestRPM)) if newestRPM == -1: newer = b else: break # the local is newer if newer is not None: - info = session.getBuild("%s-%s-%s" % (str(newer['name']), newer['version'], newer['release'])) + info = session.getBuild("%s-%s-%s" % + (str(newer['name']), newer['version'], newer['release'])) if info: build = LocalBuild(info) self.substitute_idx[parentnvr] = build @@ -751,7 +760,8 @@ class BuildTracker(object): if depth > 0: log("%sDep replaced: %s->%s" % (head, build.nvr, replace)) return build - if options.prefer_new and (depth > 0) and (tag is not None) and not (build.state == "common"): + if options.prefer_new and (depth > 0) and (tag is not None) and \ + not (build.state == "common"): latestBuild = self.newerBuild(build, tag) if latestBuild is not None: build.substitute = latestBuild.nvr @@ -875,7 +885,8 @@ class BuildTracker(object): finally: os.umask(old_umask) else: - # TODO - would be possible, using uploadFile directly, to upload without writing locally. + # TODO - would be possible, using uploadFile directly, + # to upload without writing locally. # for now, though, just use uploadWrapper koji.ensuredir(options.workpath) dst = "%s/%s" % (options.workpath, fn) @@ -1053,7 +1064,8 @@ class BuildTracker(object): session.groupListAdd(taginfo['id'], 'build', force=True) # using force in case group is blocked. This shouldn't be the case, but... for pkg_name in drop_pkgs: - # in principal, our tag should not have inheritance, so the remove call is the right thing + # in principal, our tag should not have inheritance, + # so the remove call is the right thing session.groupPackageListRemove(taginfo['id'], 'build', pkg_name) for pkg_name in add_pkgs: session.groupPackageListAdd(taginfo['id'], 'build', pkg_name) @@ -1278,7 +1290,8 @@ def main(args): logfile = None if logfile is not None: log("logging to %s" % filename) - os.write(logfile, "\n\n========================================================================\n") + os.write(logfile, + "\n\n========================================================================\n") if options.build: binfo = remote.getBuild(options.build, strict=True) diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 8714e85..48da704 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -36,7 +36,8 @@ def clean_reservations(cursor, vacuum, test, age): def clean_notification_tasks(cursor, vacuum, test, age): - q = " FROM task WHERE method = 'build' AND completion_time < NOW() - '%s days'::interval" % int(age) + q = " FROM task WHERE method = 'build' AND completion_time < NOW() - '%s days'::interval" % \ + int(age) if options.verbose: cursor.execute("SELECT COUNT(*) " + q) rows = cursor.fetchall()[0][0] @@ -95,7 +96,8 @@ def clean_scratch_tasks(cursor, vacuum, test, age): return # delete standard buildroots - cursor.execute("DELETE FROM standard_buildroot WHERE task_id IN (SELECT task_id FROM temp_scratch_tasks)") + cursor.execute( + "DELETE FROM standard_buildroot WHERE task_id IN (SELECT task_id FROM temp_scratch_tasks)") # delete tasks finally cursor.execute("DELETE FROM task WHERE id IN (SELECT task_id FROM temp_scratch_tasks)") @@ -106,7 +108,8 @@ def clean_scratch_tasks(cursor, vacuum, test, age): def clean_buildroots(cursor, vacuum, test): - q = " FROM buildroot WHERE cg_id IS NULL AND id NOT IN (SELECT buildroot_id FROM standard_buildroot)" + q = " FROM buildroot " \ + "WHERE cg_id IS NULL AND id NOT IN (SELECT buildroot_id FROM standard_buildroot)" if options.verbose: cursor.execute("SELECT COUNT(*) " + q) @@ -206,7 +209,8 @@ if __name__ == "__main__": clean_sessions(cursor, options.vacuum, options.test, options.sessions_age) clean_reservations(cursor, options.vacuum, options.test, options.reservations_age) if options.tag_notifications: - clean_notification_tasks(cursor, options.vacuum, options.test, age=options.tag_notifications_age) + clean_notification_tasks(cursor, options.vacuum, options.test, + age=options.tag_notifications_age) if options.scratch: clean_scratch_tasks(cursor, options.vacuum, options.test, age=options.scratch_age) if options.buildroots: diff --git a/util/kojira b/util/kojira index 3931bb9..8ad82ce 100755 --- a/util/kojira +++ b/util/kojira @@ -269,7 +269,8 @@ class RepoManager(object): self._local.session = value def printState(self): - self.logger.debug('Tracking %i repos, %i child processes', len(self.repos), len(self.delete_pids)) + self.logger.debug('Tracking %i repos, %i child processes', + len(self.repos), len(self.delete_pids)) for tag_id, task_id in six.iteritems(self.tasks): self.logger.debug("Tracking task %s for tag %s", task_id, tag_id) for pid, desc in six.iteritems(self.delete_pids): @@ -348,8 +349,9 @@ class RepoManager(object): if repo: # we're already tracking it if repo.state != data['state']: - self.logger.info('State changed for repo %s: %s -> %s' - % (repo_id, koji.REPO_STATES[repo.state], koji.REPO_STATES[data['state']])) + self.logger.info( + 'State changed for repo %s: %s -> %s', + repo_id, koji.REPO_STATES[repo.state], koji.REPO_STATES[data['state']]) repo.state = data['state'] else: self.logger.info('Found repo %s, state=%s' @@ -357,7 +359,7 @@ class RepoManager(object): repo = ManagedRepo(self, data) self.repos[repo_id] = repo if not getTag(self.session, repo.tag_id) and not repo.expired(): - self.logger.info('Tag %d for repo %d disappeared, expiring.' % (repo.tag_id, repo_id)) + self.logger.info('Tag %d for repo %d disappeared, expiring.', repo.tag_id, repo_id) repo.expire() if len(self.repos) > len(repodata): # This shouldn't normally happen, but might if someone else calls @@ -491,20 +493,23 @@ class RepoManager(object): self.logger.debug("did not expect %s; age: %s", repodir, age) if age > max_age: - self.logger.info("Removing unexpected directory (no such repo): %s", repodir) + self.logger.info( + "Removing unexpected directory (no such repo): %s", repodir) if symlink: os.unlink(repodir) else: self.rmtree(repodir) continue if rinfo['tag_name'] != tag: - self.logger.warn("Tag name mismatch (rename?): %s vs %s", tag, rinfo['tag_name']) + self.logger.warn( + "Tag name mismatch (rename?): %s vs %s", tag, rinfo['tag_name']) continue if rinfo['state'] in (koji.REPO_DELETED, koji.REPO_PROBLEM): age = time.time() - max(rinfo['create_ts'], dir_ts) self.logger.debug("potential removal candidate: %s; age: %s" % (repodir, age)) if age > max_age: - logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) + logger.info("Removing stray repo (state=%s): %s", + koji.REPO_STATES[rinfo['state']], repodir) if symlink: os.unlink(repodir) else: @@ -622,11 +627,12 @@ class RepoManager(object): tstate = koji.TASK_STATES[tinfo['state']] tag_id = self.tasks[task_id]['tag_id'] if tstate == 'CLOSED': - self.logger.info("Finished: newRepo task %s for tag %s" % (task_id, tag_id)) + self.logger.info("Finished: newRepo task %s for tag %s", task_id, tag_id) self.recent_tasks[task_id] = time.time() del self.tasks[task_id] elif tstate in ('CANCELED', 'FAILED'): - self.logger.info("Problem: newRepo task %s for tag %s is %s" % (task_id, tag_id, tstate)) + self.logger.info( + "Problem: newRepo task %s for tag %s is %s", task_id, tag_id, tstate) self.recent_tasks[task_id] = time.time() del self.tasks[task_id] else: @@ -635,7 +641,8 @@ class RepoManager(object): # also check other newRepo tasks repo_tasks = self.session.listTasks(opts={'method': 'newRepo', - 'state': ([koji.TASK_STATES[s] for s in ('FREE', 'OPEN')])}) + 'state': ([koji.TASK_STATES[s] + for s in ('FREE', 'OPEN')])}) others = [t for t in repo_tasks if t['id'] not in self.tasks] for tinfo in others: if tinfo['id'] not in self.other_tasks: @@ -947,8 +954,8 @@ def get_options(): 'max_delete_processes', 'max_repo_tasks_maven', 'delete_batch_size', 'dist_repo_lifetime', 'sleeptime', 'recent_tasks_lifetime') - str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', - 'cert', 'ca', 'serverca', 'debuginfo_tags', + str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', + 'krbservice', 'cert', 'ca', 'serverca', 'debuginfo_tags', 'source_tags', 'separate_source_tags', 'ignore_tags') # FIXME: remove ca here bool_opts = ('verbose', 'debug', 'ignore_stray_repos', 'offline_retry', 'krb_rdns', 'krb_canon_host', 'no_ssl_verify') diff --git a/vm/kojikamid.py b/vm/kojikamid.py index a2f159e..d03d9ad 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -183,17 +183,20 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" src_scm = SCM(self.source_url) # noqa: F821 - self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) # noqa: F821 + self.source_dir = src_scm.checkout( + ensuredir(os.path.join(self.workdir, 'source'))) # noqa: F821 self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: spec_scm = SCM(self.task_opts['winspec']) # noqa: F821 - self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) # noqa: F821 + self.spec_dir = spec_scm.checkout( + ensuredir(os.path.join(self.workdir, 'spec'))) # noqa: F821 self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) else: self.spec_dir = self.source_dir if 'patches' in self.task_opts: patch_scm = SCM(self.task_opts['patches']) # noqa: F821 - self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) # noqa: F821 + self.patches_dir = patch_scm.checkout( + ensuredir(os.path.join(self.workdir, 'patches'))) # noqa: F821 self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) self.applyPatches(self.source_dir, self.patches_dir) self.virusCheck(self.workdir) @@ -207,7 +210,8 @@ class WindowsBuild(object): raise BuildError('no patches found at %s' % patchdir) # noqa: F821 patches.sort() for patch in patches: - cmd = ['/bin/patch', '--verbose', '-d', sourcedir, '-p1', '-i', os.path.join(patchdir, patch)] + cmd = ['/bin/patch', '--verbose', '-d', sourcedir, '-p1', '-i', + os.path.join(patchdir, patch)] run(cmd, fatal=True) def loadConfig(self): @@ -241,7 +245,8 @@ class WindowsBuild(object): # absolute paths, or without a path in which case it is searched for # on the PATH. if conf.has_option('building', 'preinstalled'): - self.preinstalled.extend([e.strip() for e in conf.get('building', 'preinstalled').split('\n') if e]) + self.preinstalled.extend( + [e.strip() for e in conf.get('building', 'preinstalled').split('\n') if e]) # buildrequires and provides are multi-valued (space-separated) for br in conf.get('building', 'buildrequires').split(): @@ -336,7 +341,8 @@ class WindowsBuild(object): with open(destpath, 'w') as destfile: offset = 0 while True: - encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, brtype) + encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, + brtype) if not encoded: break data = base64.b64decode(encoded) @@ -349,9 +355,11 @@ class WindowsBuild(object): if 'checksum_type' in fileinfo: digest = checksum.hexdigest() if fileinfo['checksum'] != digest: - raise BuildError('checksum validation failed for %s, %s (computed) != %s (provided)' % # noqa: F821 - (destpath, digest, fileinfo['checksum'])) - self.logger.info('Retrieved %s (%s bytes, %s: %s)', destpath, offset, checksum_type, digest) + raise BuildError( # noqa: F821 + 'checksum validation failed for %s, %s (computed) != %s (provided)' % + (destpath, digest, fileinfo['checksum'])) + self.logger.info( + 'Retrieved %s (%s bytes, %s: %s)', destpath, offset, checksum_type, digest) else: self.logger.info('Retrieved %s (%s bytes)', destpath, offset) @@ -409,7 +417,8 @@ class WindowsBuild(object): def cmdBuild(self): """Do the build: run the execute line(s) with cmd.exe""" - tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat', dir='/cygdrive/c/Windows/Temp') + tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat', + dir='/cygdrive/c/Windows/Temp') script = os.fdopen(tmpfd, 'w') for attr in ['source_dir', 'spec_dir', 'patches_dir']: val = getattr(self, attr) @@ -630,7 +639,8 @@ def get_mgmt_server(): # supported by python/cygwin/Windows task_port = server.getPort(macaddr) logger.debug('found task-specific port %s', task_port) - return six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + return six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, task_port), + allow_none=True) def get_options(): @@ -641,8 +651,10 @@ def get_options(): """ parser = OptionParser(usage=usage) parser.add_option('-d', '--debug', action='store_true', help='Log debug statements') - parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False) - parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False) + parser.add_option('-i', '--install', action='store_true', default=False, + help='Install this daemon as a service') + parser.add_option('-u', '--uninstall', action='store_true', default=False, + help='Uninstall this daemon if it was installed previously as a service') (options, args) = parser.parse_args() return options diff --git a/vm/kojivmd b/vm/kojivmd index ef9fa2b..3981783 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -269,9 +269,11 @@ class DaemonXMLRPCServer(six.moves.xmlrpc_server.SimpleXMLRPCServer): def __init__(self, addr, port): if sys.version_info[:2] <= (2, 4): - six.moves.xmlrpc_server.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False) + six.moves.xmlrpc_server.SimpleXMLRPCServer.__init__(self, (addr, port), + logRequests=False) else: - six.moves.xmlrpc_server.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False, + six.moves.xmlrpc_server.SimpleXMLRPCServer.__init__(self, (addr, port), + logRequests=False, allow_none=True) self.logger = logging.getLogger('koji.vm.DaemonXMLRPCServer') self.socket.settimeout(5) @@ -307,7 +309,8 @@ class DaemonXMLRPCServer(six.moves.xmlrpc_server.SimpleXMLRPCServer): else: response = self._dispatch(method, params) response = (response,) - response = six.moves.xmlrpc_client.dumps(response, methodresponse=1, allow_none=True) + response = six.moves.xmlrpc_client.dumps(response, + methodresponse=1, allow_none=True) except six.moves.xmlrpc_client.Fault as fault: response = six.moves.xmlrpc_client.dumps(fault) except BaseException: @@ -369,7 +372,9 @@ class WinBuildTask(MultiPlatformTask): task_opts = koji.util.dslice(opts, ['timeout', 'cpus', 'mem', 'static_mac'], strict=False) task_id = self.session.host.subtask(method='vmExec', - arglist=[name, [source_url, build_tag['name'], subopts], task_opts], + arglist=[name, + [source_url, build_tag['name'], subopts], + task_opts], label=name[:255], parent=self.id) results = self.wait(task_id)[task_id] @@ -379,7 +384,8 @@ class WinBuildTask(MultiPlatformTask): if not opts.get('scratch'): build_info = koji.util.dslice(results, ['name', 'version', 'release', 'epoch']) build_info['package_name'] = build_info['name'] - pkg_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'], event=event_id) + pkg_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'], + event=event_id) if not opts.get('skip_tag'): # Make sure package is on the list for this tag if pkg_cfg is None: @@ -397,8 +403,8 @@ class WinBuildTask(MultiPlatformTask): rpm_results = None spec_url = opts.get('specfile') if spec_url: - rpm_results = self.buildWrapperRPM(spec_url, task_id, target_info, build_info, repo_id, - channel='default') + rpm_results = self.buildWrapperRPM(spec_url, task_id, target_info, build_info, + repo_id, channel='default') if opts.get('scratch'): self.session.host.moveWinBuildToScratch(self.id, results, rpm_results) @@ -436,8 +442,8 @@ class VMExecTask(BaseTaskHandler): def __init__(self, *args, **kw): super(VMExecTask, self).__init__(*args, **kw) - self.task_manager = six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (self.options.privaddr, self.options.portbase), - allow_none=True) + self.task_manager = six.moves.xmlrpc_client.ServerProxy( + 'http://%s:%s/' % (self.options.privaddr, self.options.portbase), allow_none=True) self.port = None self.server = None self.task_info = None @@ -451,13 +457,16 @@ class VMExecTask(BaseTaskHandler): def mkqcow2(self, clone_name, source_disk, disk_num): new_name = clone_name + '-disk-' + str(disk_num) + self.QCOW2_EXT new_path = os.path.join(self.options.imagedir, new_name) - cmd = ['/usr/bin/qemu-img', 'create', '-f', 'qcow2', '-o', 'backing_file=%s' % source_disk, new_path] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + cmd = ['/usr/bin/qemu-img', 'create', '-f', 'qcow2', '-o', 'backing_file=%s' % source_disk, + new_path] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) output, dummy = proc.communicate() ret = proc.wait() if ret: - raise koji.BuildError('unable to create qcow2 image, "%s" returned %s; output was: %s' % - (' '.join(cmd), ret, output)) + raise koji.BuildError( + 'unable to create qcow2 image, "%s" returned %s; output was: %s' % + (' '.join(cmd), ret, output)) vm_user = pwd.getpwnam(self.options.vmuser) os.chown(new_path, vm_user.pw_uid, vm_user.pw_gid) return new_path @@ -708,14 +717,17 @@ class VMExecTask(BaseTaskHandler): hdr = koji.get_rpm_header(localpath) payloadhash = koji.hex_string(koji.get_header_field(hdr, 'sigmd5')) if fileinfo['payloadhash'] != payloadhash: - raise koji.BuildError("Downloaded rpm %s doesn't match checksum (expected: %s, got %s)" % ( - os.path.basename(fileinfo['localpath']), - fileinfo['payloadhash'], payloadhash)) + raise koji.BuildError( + "Downloaded rpm %s doesn't match checksum (expected: %s, got %s)" % + (os.path.basename(fileinfo['localpath']), + fileinfo['payloadhash'], + payloadhash)) if not koji.util.check_sigmd5(localpath): raise koji.BuildError("Downloaded rpm %s doesn't match sigmd5" % os.path.basename(fileinfo['localpath'])) else: - self.verifyChecksum(localpath, fileinfo['checksum'], koji.CHECKSUM_TYPES[fileinfo['checksum_type']]) + self.verifyChecksum(localpath, fileinfo['checksum'], + koji.CHECKSUM_TYPES[fileinfo['checksum_type']]) return open(localpath, 'r') @@ -796,8 +808,9 @@ class VMExecTask(BaseTaskHandler): if sum.hexdigest() == checksum: return True else: - raise koji.BuildError('%s checksum validation failed for %s, %s (computed) != %s (provided)' % - (algo, local_path, sum.hexdigest(), checksum)) + raise koji.BuildError( + '%s checksum validation failed for %s, %s (computed) != %s (provided)' % + (algo, local_path, sum.hexdigest(), checksum)) def closeTask(self, output): self.output = output @@ -879,8 +892,9 @@ class VMExecTask(BaseTaskHandler): if mins > timeout: vm.destroy() self.server.server_close() - raise koji.BuildError('Task did not complete after %.2f minutes, VM %s has been destroyed' % - (mins, clone_name)) + raise koji.BuildError( + 'Task did not complete after %.2f minutes, VM %s has been destroyed' % + (mins, clone_name)) else: vm.destroy() self.server.server_close() @@ -913,7 +927,9 @@ class VMTaskManager(TaskManager): if macaddr in self.macaddrs: raise koji.PreBuildError('duplicate MAC address: %s' % macaddr) self.macaddrs[macaddr] = (vm_name, task_id, port) - self.logger.info('registered MAC address %s for VM %s (task ID %s, port %s)', macaddr, vm_name, task_id, port) + self.logger.info( + 'registered MAC address %s for VM %s (task ID %s, port %s)', + macaddr, vm_name, task_id, port) return True finally: self.macaddr_lock.release() @@ -964,7 +980,8 @@ class VMTaskManager(TaskManager): availableMB = available // 1024 // 1024 self.logger.debug('disk space available in %s: %i MB', self.options.imagedir, availableMB) if availableMB < self.options.minspace: - self.status = 'Insufficient disk space: %i MB, %i MB required' % (availableMB, self.options.minspace) + self.status = 'Insufficient disk space: %i MB, %i MB required' % \ + (availableMB, self.options.minspace) self.logger.warn(self.status) return False return True diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index eb4e881..f62e96c 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -154,9 +154,12 @@ def _assertLogin(environ): raise koji.AuthError('could not login %s via SSL' % environ['koji.currentLogin']) elif options['WebPrincipal']: if not _krbLogin(environ, environ['koji.session'], environ['koji.currentLogin']): - raise koji.AuthError('could not login using principal: %s' % environ['koji.currentLogin']) + raise koji.AuthError( + 'could not login using principal: %s' % environ['koji.currentLogin']) else: - raise koji.AuthError('KojiWeb is incorrectly configured for authentication, contact the system administrator') + raise koji.AuthError( + 'KojiWeb is incorrectly configured for authentication, ' + 'contact the system administrator') # verify a valid authToken was passed in to avoid CSRF authToken = environ['koji.form'].getfirst('a', '') @@ -168,7 +171,8 @@ def _assertLogin(environ): # their authToken is likely expired # send them back to the page that brought them here so they # can re-click the link with a valid authToken - _redirectBack(environ, page=None, forceSSL=(_getBaseURL(environ).startswith('https://'))) + _redirectBack(environ, page=None, + forceSSL=(_getBaseURL(environ).startswith('https://'))) assert False # pragma: no cover else: _redirect(environ, 'login') @@ -188,7 +192,8 @@ def _getServer(environ): if environ['koji.currentLogin']: environ['koji.currentUser'] = session.getUser(environ['koji.currentLogin']) if not environ['koji.currentUser']: - raise koji.AuthError('could not get user for principal: %s' % environ['koji.currentLogin']) + raise koji.AuthError( + 'could not get user for principal: %s' % environ['koji.currentLogin']) _setUserCookie(environ, environ['koji.currentLogin']) else: environ['koji.currentUser'] = None @@ -271,7 +276,9 @@ def login(environ, page=None): elif options['WebPrincipal']: principal = environ.get('REMOTE_USER') if not principal: - raise koji.AuthError('configuration error: mod_auth_gssapi should have performed authentication before presenting this page') + raise koji.AuthError( + 'configuration error: mod_auth_gssapi should have performed authentication before ' + 'presenting this page') if not _krbLogin(environ, session, principal): raise koji.AuthError('could not login using principal: %s' % principal) @@ -279,7 +286,9 @@ def login(environ, page=None): username = principal authlogger.info('Successful Kerberos authentication by %s', username) else: - raise koji.AuthError('KojiWeb is incorrectly configured for authentication, contact the system administrator') + raise koji.AuthError( + 'KojiWeb is incorrectly configured for authentication, contact the system ' + 'administrator') _setUserCookie(environ, username) # To protect the session cookie, we must forceSSL @@ -322,8 +331,10 @@ def index(environ, packageOrder='package_name', packageStart=None): values['order'] = '-id' if user: - kojiweb.util.paginateResults(server, values, 'listPackages', kw={'userID': user['id'], 'with_dups': True}, - start=packageStart, dataName='packages', prefix='package', order=packageOrder, pageSize=10) + kojiweb.util.paginateResults(server, values, 'listPackages', + kw={'userID': user['id'], 'with_dups': True}, + start=packageStart, dataName='packages', prefix='package', + order=packageOrder, pageSize=10) notifs = server.getBuildNotifications(user['id']) notifs.sort(key=lambda x: x['id']) @@ -480,12 +491,16 @@ _TASKS = ['build', 'livemedia', 'createLiveMedia'] # Tasks that can exist without a parent -_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'distRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] +_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', + 'winbuild', 'newRepo', 'distRepo', 'tagBuild', 'tagNotification', 'waitrepo', + 'livecd', 'appliance', 'image', 'livemedia'] # Tasks that can have children -_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'distRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] +_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'distRepo', + 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] -def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, channelID=None, start=None, order='-id'): +def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, + channelID=None, start=None, order='-id'): values = _initValues(environ, 'Tasks', 'tasks') server = _getServer(environ) @@ -539,7 +554,9 @@ def tasks(environ, owner=None, state='active', view='tree', method='all', hostID opts['parent'] = None if state == 'active': - opts['state'] = [koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN'], koji.TASK_STATES['ASSIGNED']] + opts['state'] = [koji.TASK_STATES['FREE'], + koji.TASK_STATES['OPEN'], + koji.TASK_STATES['ASSIGNED']] elif state == 'all': pass else: @@ -830,7 +847,8 @@ def _chunk_file(server, environ, taskID, name, offset, size, volume): chunk_size = 1048576 if remaining < chunk_size: chunk_size = remaining - content = server.downloadTaskOutput(taskID, name, offset=offset, size=chunk_size, volume=volume) + content = server.downloadTaskOutput(taskID, name, + offset=offset, size=chunk_size, volume=volume) if not content: break yield content @@ -863,7 +881,8 @@ def tags(environ, start=None, order=None, childID=None): _PREFIX_CHARS = [chr(char) for char in list(range(48, 58)) + list(range(97, 123))] -def packages(environ, tagID=None, userID=None, order='package_name', start=None, prefix=None, inherited='1'): +def packages(environ, tagID=None, userID=None, order='package_name', start=None, prefix=None, + inherited='1'): values = _initValues(environ, 'Packages', 'packages') server = _getServer(environ) tag = None @@ -890,7 +909,10 @@ def packages(environ, tagID=None, userID=None, order='package_name', start=None, values['inherited'] = inherited kojiweb.util.paginateMethod(server, values, 'listPackages', - kw={'tagID': tagID, 'userID': userID, 'prefix': prefix, 'inherited': bool(inherited)}, + kw={'tagID': tagID, + 'userID': userID, + 'prefix': prefix, + 'inherited': bool(inherited)}, start=start, dataName='packages', prefix='package', order=order) values['chars'] = _PREFIX_CHARS @@ -898,7 +920,8 @@ def packages(environ, tagID=None, userID=None, order='package_name', start=None, return _genHTML(environ, 'packages.chtml') -def packageinfo(environ, packageID, tagOrder='name', tagStart=None, buildOrder='-completion_time', buildStart=None): +def packageinfo(environ, packageID, tagOrder='name', tagStart=None, buildOrder='-completion_time', + buildStart=None): values = _initValues(environ, 'Package Info', 'packages') server = _getServer(environ) @@ -916,12 +939,14 @@ def packageinfo(environ, packageID, tagOrder='name', tagStart=None, buildOrder=' kojiweb.util.paginateMethod(server, values, 'listTags', kw={'package': package['id']}, start=tagStart, dataName='tags', prefix='tag', order=tagOrder) kojiweb.util.paginateMethod(server, values, 'listBuilds', kw={'packageID': package['id']}, - start=buildStart, dataName='builds', prefix='build', order=buildOrder) + start=buildStart, dataName='builds', prefix='build', + order=buildOrder) return _genHTML(environ, 'packageinfo.chtml') -def taginfo(environ, tagID, all='0', packageOrder='package_name', packageStart=None, buildOrder='-completion_time', buildStart=None, childID=None): +def taginfo(environ, tagID, all='0', packageOrder='package_name', packageStart=None, + buildOrder='-completion_time', buildStart=None, childID=None): values = _initValues(environ, 'Tag Info', 'tags') server = _getServer(environ) @@ -1115,7 +1140,9 @@ def tagparent(environ, tagID, parentID, action): elif len(inheritanceData) == 1: values['inheritanceData'] = inheritanceData[0] else: - raise koji.GenericError('tag %i has tag %i listed as a parent more than once' % (tag['id'], parent['id'])) + raise koji.GenericError( + 'tag %i has tag %i listed as a parent more than once' % + (tag['id'], parent['id'])) return _genHTML(environ, 'tagparent.chtml') elif action == 'remove': @@ -1174,7 +1201,8 @@ def buildinfo(environ, buildID): for archive in archives: if btype == 'maven': archive['display'] = archive['filename'] - archive['dl_url'] = '/'.join([pathinfo.mavenbuild(build), pathinfo.mavenfile(archive)]) + archive['dl_url'] = '/'.join([pathinfo.mavenbuild(build), + pathinfo.mavenfile(archive)]) elif btype == 'win': archive['display'] = pathinfo.winfile(archive) archive['dl_url'] = '/'.join([pathinfo.winbuild(build), pathinfo.winfile(archive)]) @@ -1210,7 +1238,8 @@ def buildinfo(environ, buildID): # get the summary, description, and changelogs from the built srpm # if the build is not yet complete if build['state'] != koji.BUILD_STATES['COMPLETE']: - srpm_tasks = server.listTasks(opts={'parent': task['id'], 'method': 'buildSRPMFromSCM'}) + srpm_tasks = server.listTasks(opts={'parent': task['id'], + 'method': 'buildSRPMFromSCM'}) if srpm_tasks: srpm_task = srpm_tasks[0] if srpm_task['state'] == koji.TASK_STATES['CLOSED']: @@ -1220,12 +1249,14 @@ def buildinfo(environ, buildID): srpm_path = output break if srpm_path: - srpm_headers = server.getRPMHeaders(taskID=srpm_task['id'], filepath=srpm_path, + srpm_headers = server.getRPMHeaders(taskID=srpm_task['id'], + filepath=srpm_path, headers=['summary', 'description']) if srpm_headers: values['summary'] = koji.fixEncoding(srpm_headers['summary']) values['description'] = koji.fixEncoding(srpm_headers['description']) - changelog = server.getChangelogEntries(taskID=srpm_task['id'], filepath=srpm_path) + changelog = server.getChangelogEntries(taskID=srpm_task['id'], + filepath=srpm_path) if changelog: values['changelog'] = changelog else: @@ -1276,7 +1307,8 @@ def buildinfo(environ, buildID): return _genHTML(environ, 'buildinfo.chtml') -def builds(environ, userID=None, tagID=None, packageID=None, state=None, order='-build_id', start=None, prefix=None, inherited='1', latest='1', type=None): +def builds(environ, userID=None, tagID=None, packageID=None, state=None, order='-build_id', + start=None, prefix=None, inherited='1', latest='1', type=None): values = _initValues(environ, 'Builds', 'builds') server = _getServer(environ) @@ -1344,15 +1376,20 @@ def builds(environ, userID=None, tagID=None, packageID=None, state=None, order=' if tag: # don't need to consider 'state' here, since only completed builds would be tagged - kojiweb.util.paginateResults(server, values, 'listTagged', kw={'tag': tag['id'], 'package': (package and package['name'] or None), - 'owner': (user and user['name'] or None), - 'type': type, - 'inherit': bool(inherited), 'latest': bool(latest), 'prefix': prefix}, + kojiweb.util.paginateResults(server, values, 'listTagged', + kw={'tag': tag['id'], + 'package': (package and package['name'] or None), + 'owner': (user and user['name'] or None), + 'type': type, + 'inherit': bool(inherited), 'latest': bool(latest), + 'prefix': prefix}, start=start, dataName='builds', prefix='build', order=order) else: - kojiweb.util.paginateMethod(server, values, 'listBuilds', kw={'userID': (user and user['id'] or None), 'packageID': (package and package['id'] or None), - 'type': type, - 'state': state, 'prefix': prefix}, + kojiweb.util.paginateMethod(server, values, 'listBuilds', + kw={'userID': (user and user['id'] or None), + 'packageID': (package and package['id'] or None), + 'type': type, + 'state': state, 'prefix': prefix}, start=start, dataName='builds', prefix='build', order=order) values['chars'] = _PREFIX_CHARS @@ -1380,7 +1417,8 @@ def users(environ, order='name', start=None, prefix=None): return _genHTML(environ, 'users.chtml') -def userinfo(environ, userID, packageOrder='package_name', packageStart=None, buildOrder='-completion_time', buildStart=None): +def userinfo(environ, userID, packageOrder='package_name', packageStart=None, + buildOrder='-completion_time', buildStart=None): values = _initValues(environ, 'User Info', 'users') server = _getServer(environ) @@ -1392,18 +1430,23 @@ def userinfo(environ, userID, packageOrder='package_name', packageStart=None, bu values['user'] = user values['userID'] = userID - values['taskCount'] = server.listTasks(opts={'owner': user['id'], 'parent': None}, queryOpts={'countOnly': True}) + values['taskCount'] = server.listTasks(opts={'owner': user['id'], 'parent': None}, + queryOpts={'countOnly': True}) - kojiweb.util.paginateResults(server, values, 'listPackages', kw={'userID': user['id'], 'with_dups': True}, - start=packageStart, dataName='packages', prefix='package', order=packageOrder, pageSize=10) + kojiweb.util.paginateResults(server, values, 'listPackages', + kw={'userID': user['id'], 'with_dups': True}, + start=packageStart, dataName='packages', prefix='package', + order=packageOrder, pageSize=10) kojiweb.util.paginateMethod(server, values, 'listBuilds', kw={'userID': user['id']}, - start=buildStart, dataName='builds', prefix='build', order=buildOrder, pageSize=10) + start=buildStart, dataName='builds', prefix='build', + order=buildOrder, pageSize=10) return _genHTML(environ, 'userinfo.chtml') -def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-id', buildrootStart=None): +def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-id', + buildrootStart=None): values = _initValues(environ, 'RPM Info', 'builds') server = _getServer(environ) @@ -1441,8 +1484,11 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i values['summary'] = koji.fixEncoding(headers.get('summary')) values['description'] = koji.fixEncoding(headers.get('description')) values['license'] = koji.fixEncoding(headers.get('license')) - buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']}, - start=buildrootStart, dataName='buildroots', prefix='buildroot', + buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', + kw={'rpmID': rpm['id']}, + start=buildrootStart, + dataName='buildroots', + prefix='buildroot', order=buildrootOrder) values['rpmID'] = rpmID @@ -1457,7 +1503,8 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i return _genHTML(environ, 'rpminfo.chtml') -def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootOrder='-id', buildrootStart=None): +def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootOrder='-id', + buildrootStart=None): values = _initValues(environ, 'Archive Info', 'builds') server = _getServer(environ) @@ -1476,8 +1523,11 @@ def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootO builtInRoot = server.getBuildroot(archive['buildroot_id']) kojiweb.util.paginateMethod(server, values, 'listArchiveFiles', args=[archive['id']], start=fileStart, dataName='files', prefix='file', order=fileOrder) - buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'archiveID': archive['id']}, - start=buildrootStart, dataName='buildroots', prefix='buildroot', + buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', + kw={'archiveID': archive['id']}, + start=buildrootStart, + dataName='buildroots', + prefix='buildroot', order=buildrootOrder) values['title'] = archive['filename'] + ' | Archive Info' @@ -1491,7 +1541,8 @@ def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootO values['builtInRoot'] = builtInRoot values['buildroots'] = buildroots values['show_rpm_components'] = server.listRPMs(imageID=archive['id'], queryOpts={'limit': 1}) - values['show_archive_components'] = server.listArchives(imageID=archive['id'], queryOpts={'limit': 1}) + values['show_archive_components'] = server.listArchives(imageID=archive['id'], + queryOpts={'limit': 1}) return _genHTML(environ, 'archiveinfo.chtml') @@ -1604,7 +1655,8 @@ def hostinfo(environ, hostID=None, userID=None): channels = server.listChannels(host['id']) channels.sort(key=_sortbyname) buildroots = server.listBuildroots(hostID=host['id'], - state=[state[1] for state in koji.BR_STATES.items() if state[0] != 'EXPIRED']) + state=[state[1] for state in koji.BR_STATES.items() + if state[0] != 'EXPIRED']) buildroots.sort(key=lambda x: x['create_event_time'], reverse=True) values['host'] = host @@ -1718,7 +1770,8 @@ def channelinfo(environ, channelID): return _genHTML(environ, 'channelinfo.chtml') -def buildrootinfo(environ, buildrootID, builtStart=None, builtOrder=None, componentStart=None, componentOrder=None): +def buildrootinfo(environ, buildrootID, builtStart=None, builtOrder=None, componentStart=None, + componentOrder=None): values = _initValues(environ, 'Buildroot Info', 'hosts') server = _getServer(environ) @@ -1807,11 +1860,15 @@ def archivelist(environ, type, buildrootID=None, imageID=None, start=None, order raise koji.GenericError('unknown buildroot ID: %i' % buildrootID) if type == 'component': - kojiweb.util.paginateMethod(server, values, 'listArchives', kw={'componentBuildrootID': buildroot['id']}, - start=start, dataName='archives', prefix='archive', order=order) + kojiweb.util.paginateMethod(server, values, 'listArchives', + kw={'componentBuildrootID': buildroot['id']}, + start=start, dataName='archives', prefix='archive', + order=order) elif type == 'built': - kojiweb.util.paginateMethod(server, values, 'listArchives', kw={'buildrootID': buildroot['id']}, - start=start, dataName='archives', prefix='archive', order=order) + kojiweb.util.paginateMethod(server, values, 'listArchives', + kw={'buildrootID': buildroot['id']}, + start=start, dataName='archives', prefix='archive', + order=order) else: raise koji.GenericError('unrecognized type of archivelist') elif imageID is not None: @@ -1820,7 +1877,8 @@ def archivelist(environ, type, buildrootID=None, imageID=None, start=None, order # If/When future image types are supported, add elifs here if needed. if type == 'image': kojiweb.util.paginateMethod(server, values, 'listArchives', kw={'imageID': imageID}, - start=start, dataName='archives', prefix='archive', order=order) + start=start, dataName='archives', prefix='archive', + order=order) else: raise koji.GenericError('unrecognized type of archivelist') else: @@ -2155,9 +2213,12 @@ def buildsbystatus(environ, days='7'): server.multicall = True # use taskID=-1 to filter out builds with a null task_id (imported rather than built in koji) - server.listBuilds(completeAfter=dateAfter, state=koji.BUILD_STATES['COMPLETE'], taskID=-1, queryOpts={'countOnly': True}) - server.listBuilds(completeAfter=dateAfter, state=koji.BUILD_STATES['FAILED'], taskID=-1, queryOpts={'countOnly': True}) - server.listBuilds(completeAfter=dateAfter, state=koji.BUILD_STATES['CANCELED'], taskID=-1, queryOpts={'countOnly': True}) + server.listBuilds(completeAfter=dateAfter, state=koji.BUILD_STATES['COMPLETE'], taskID=-1, + queryOpts={'countOnly': True}) + server.listBuilds(completeAfter=dateAfter, state=koji.BUILD_STATES['FAILED'], taskID=-1, + queryOpts={'countOnly': True}) + server.listBuilds(completeAfter=dateAfter, state=koji.BUILD_STATES['CANCELED'], taskID=-1, + queryOpts={'countOnly': True}) [[numSucceeded], [numFailed], [numCanceled]] = server.multiCall() values['numSucceeded'] = numSucceeded @@ -2298,7 +2359,8 @@ def recentbuilds(environ, user=None, tag=None, package=None): packageObj = server.getPackage(package) if tagObj is not None: - builds = server.listTagged(tagObj['id'], inherit=True, package=(packageObj and packageObj['name'] or None), + builds = server.listTagged(tagObj['id'], inherit=True, + package=(packageObj and packageObj['name'] or None), owner=(userObj and userObj['name'] or None)) builds.sort(key=kojiweb.util.sortByKeyFuncNoneGreatest('completion_time'), reverse=True) builds = builds[:20] @@ -2408,7 +2470,8 @@ def search(environ, start=None, order=None): values['order'] = order results = kojiweb.util.paginateMethod(server, values, 'search', args=(terms, type, match), - start=start, dataName='results', prefix='result', order=order) + start=start, dataName='results', prefix='result', + order=order) if not start and len(results) == 1: # if we found exactly one result, skip the result list and redirect to the info page # (you're feeling lucky) diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 96cecc8..97477ec 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -96,7 +96,9 @@ class Dispatcher(object): ['LibPath', 'string', '/usr/share/koji-web/lib'], ['LogLevel', 'string', 'WARNING'], - ['LogFormat', 'string', '%(msecs)d [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s %(name)s: %(message)s'], + ['LogFormat', 'string', + '%(msecs)d [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s ' + '%(name)s: %(message)s'], ['Tasks', 'list', []], ['ToplevelTasks', 'list', []], @@ -227,7 +229,9 @@ class Dispatcher(object): raise URLNotFound # parse form args data = {} - fs = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ.copy(), keep_blank_values=True) + fs = cgi.FieldStorage(fp=environ['wsgi.input'], + environ=environ.copy(), + keep_blank_values=True) for field in fs.list: if field.filename: val = field diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index d569298..e42a6e5 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -65,7 +65,8 @@ def _initValues(environ, title='Build System Info', pageID='summary'): themeCache.clear() themeInfo.clear() themeInfo['name'] = environ['koji.options'].get('KojiTheme', None) - themeInfo['staticdir'] = environ['koji.options'].get('KojiStaticDir', '/usr/share/koji-web/static') + themeInfo['staticdir'] = environ['koji.options'].get('KojiStaticDir', + '/usr/share/koji-web/static') environ['koji.values'] = values @@ -227,9 +228,11 @@ def sortImage(template, sortKey, orderVar='order'): """ orderVal = template.getVar(orderVar) if orderVal == sortKey: - return 'ascending sort' % themePath("images/gray-triangle-up.gif") + return 'ascending sort' % \ + themePath("images/gray-triangle-up.gif") elif orderVal == '-' + sortKey: - return 'descending sort' % themePath("images/gray-triangle-down.gif") + return 'descending sort' % \ + themePath("images/gray-triangle-down.gif") else: return '' @@ -283,7 +286,8 @@ def sortByKeyFuncNoneGreatest(key): return internal_key -def paginateList(values, data, start, dataName, prefix=None, order=None, noneGreatest=False, pageSize=50): +def paginateList(values, data, start, dataName, prefix=None, order=None, noneGreatest=False, + pageSize=50): """ Slice the 'data' list into one page worth. Start at offset 'start' and limit the total number of pages to pageSize @@ -317,8 +321,9 @@ def paginateList(values, data, start, dataName, prefix=None, order=None, noneGre def paginateMethod(server, values, methodName, args=None, kw=None, start=None, dataName=None, prefix=None, order=None, pageSize=50): - """Paginate the results of the method with the given name when called with the given args and kws. - The method must support the queryOpts keyword parameter, and pagination is done in the database.""" + """Paginate the results of the method with the given name when called with the given args and + kws. The method must support the queryOpts keyword parameter, and pagination is done in the + database.""" if args is None: args = [] if kw is None: @@ -346,10 +351,10 @@ def paginateMethod(server, values, methodName, args=None, kw=None, def paginateResults(server, values, methodName, args=None, kw=None, start=None, dataName=None, prefix=None, order=None, pageSize=50): - """Paginate the results of the method with the given name when called with the given args and kws. - This method should only be used when then method does not support the queryOpts command (because - the logic used to generate the result list prevents filtering/ordering from being done in the database). - The method must return a list of maps.""" + """Paginate the results of the method with the given name when called with the given args and + kws. This method should only be used when then method does not support the queryOpts command + (because the logic used to generate the result list prevents filtering/ordering from being done + in the database). The method must return a list of maps.""" if args is None: args = [] if kw is None: @@ -390,7 +395,8 @@ def _populateValues(values, dataName, prefix, data, totalRows, start, count, pag totalPages = int(totalRows // pageSize) if totalRows % pageSize > 0: totalPages += 1 - pages = [page for page in range(0, totalPages) if (abs(page - currentPage) < 100 or ((page + 1) % 100 == 0))] + pages = [page for page in range(0, totalPages) + if (abs(page - currentPage) < 100 or ((page + 1) % 100 == 0))] values[(prefix and prefix + 'Pages') or 'pages'] = pages From 2a2c5cb72921057642aa539ffc0697513d24fa93 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 16/23] flake8: apply W rules (prefering W503) --- diff --git a/.flake8 b/.flake8 index 932dbb0..1c4a908 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,12 @@ [flake8] -select = I,C,F,E -ignore = E266,E731 +select = E,F,W,C,I +ignore = + # too many leading ‘#’ for block comment + E266, + # do not assign a lambda expression, use a def + E731, + # line break after binary operator + W504 max_line_length = 99 exclude = .git, diff --git a/builder/kojid b/builder/kojid index 26d5dcc..1c29e63 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1046,8 +1046,8 @@ class BuildTask(BaseTaskHandler): # scratch builds do not get imported build_id = self.session.host.initBuild(data) # (initBuild raises an exception if there is a conflict) - failany = (self.opts.get('fail_fast', False) - or not getattr(self.options, 'build_arch_can_fail', False)) + failany = (self.opts.get('fail_fast', False) or + not getattr(self.options, 'build_arch_can_fail', False)) try: self.extra_information = {"src": src, "data": data, "target": target} srpm, rpms, brmap, logs = self.runBuilds(srpm, build_tag, archlist, @@ -1691,8 +1691,8 @@ class BuildMavenTask(BaseBuildTask): # Apply patches, if present if self.opts.get('patches'): # filter out directories and files beginning with . (probably scm metadata) - patches = [patch for patch in os.listdir(patchcheckoutdir) if - os.path.isfile(os.path.join(patchcheckoutdir, patch)) and + patches = [patch for patch in os.listdir(patchcheckoutdir) + if os.path.isfile(os.path.join(patchcheckoutdir, patch)) and patch.endswith('.patch')] if not patches: raise koji.BuildError('no patches found at %s' % self.opts.get('patches')) @@ -1818,7 +1818,7 @@ class WrapperRPMTask(BaseBuildTask): if re.match("%s:" % tag, spec, re.M): raise koji.BuildError("%s is not allowed to be set in spec file" % tag) for tag in ("packager", "distribution", "vendor"): - if re.match("%%define\s+%s\s+" % tag, spec, re.M): + if re.match(r"%%define\s+%s\s+" % tag, spec, re.M): raise koji.BuildError("%s is not allowed to be defined in spec file" % tag) def checkHost(self, hostdata): @@ -4563,8 +4563,8 @@ class BuildIndirectionImageTask(OzImageTask): if re.search(namere, filename): return filename - build_diskimage = _match_name(buildfiles, ".*%s\.qcow2$" % (arch)) - build_tdl = _match_name(buildfiles, "tdl.%s\.xml" % (arch)) + build_diskimage = _match_name(buildfiles, r".*%s\.qcow2$" % (arch)) + build_tdl = _match_name(buildfiles, r"tdl.%s\.xml" % (arch)) diskimage_full = os.path.join(builddir, build_diskimage) tdl_full = os.path.join(builddir, build_tdl) @@ -4859,7 +4859,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): if re.match("%s:" % tag, spec, re.M): raise koji.BuildError("%s is not allowed to be set in spec file" % tag) for tag in ("packager", "distribution", "vendor"): - if re.match("%%define\s+%s\s+" % tag, spec, re.M): + if re.match(r"%%define\s+%s\s+" % tag, spec, re.M): raise koji.BuildError("%s is not allowed to be defined in spec file" % tag) def patch_scm_source(self, sourcedir, logfile, opts): @@ -5188,8 +5188,10 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r return build_pkg_name = build['package_name'] - build_pkg_evr = '%s%s-%s' % ((build['epoch'] and str(build['epoch']) + - ':' or ''), build['version'], build['release']) + build_pkg_evr = '%s%s-%s' % \ + ((build['epoch'] and str(build['epoch']) + ':' or ''), + build['version'], + build['release']) build_nvr = koji.buildLabel(build) build_id = build['id'] build_owner = build['owner_name'] @@ -6131,8 +6133,8 @@ enabled=1 avail = to_list(rpm_idx.get(rpm_id, {}).keys()) outfile.write('%s: %r\n' % (fname, avail)) self.session.uploadWrapper(missing_log, self.uploadpath) - if (not opts['skip_missing_signatures'] - and not opts['allow_missing_signatures']): + if (not opts['skip_missing_signatures'] and + not opts['allow_missing_signatures']): raise koji.GenericError('Unsigned packages found. See ' 'missing_signatures.log') diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index b591e97..df7c95b 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -2676,8 +2676,7 @@ def anon_handle_list_groups(goptions, session, args): for x in [x[1] for x in groups]: x['tag_name'] = get_cached_tag(x['tag_id']) print_group_list_req_group(x) - pkgs = [(x['package'], x) for x in group['packagelist']] - pkgs.sort() + pkgs = sorted([(x['package'], x) for x in group['packagelist']]) for x in [x[1] for x in pkgs]: x['tag_name'] = get_cached_tag(x['tag_id']) print_group_list_req_package(x) @@ -6935,7 +6934,7 @@ def anon_handle_download_task(options, session, args): base_task = session.getTaskInfo(base_task_id) if not base_task: error(_('No such task: #%i') % base_task_id) - + def check_downloadable(task): return task["method"] == "buildArch" diff --git a/koji/__init__.py b/koji/__init__.py index 82b92c1..e4b01a1 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -988,11 +988,9 @@ def get_header_field(hdr, name, src_arch=False): if not SUPPORTED_OPT_DEP_HDRS.get(name, True): return [] - if (src_arch and name == "ARCH" - and get_header_field(hdr, "sourcepackage")): + if src_arch and name == "ARCH" and get_header_field(hdr, "sourcepackage"): # return "src" or "nosrc" arch instead of build arch for src packages - if (get_header_field(hdr, "nosource") - or get_header_field(hdr, "nopatch")): + if get_header_field(hdr, "nosource") or get_header_field(hdr, "nopatch"): return "nosrc" return "src" @@ -2172,8 +2170,8 @@ def is_requests_cert_error(e): # are way more ugly. errstr = str(e) if ('Permission denied' in errstr or # certificate not readable - 'certificate revoked' in errstr or - 'certificate expired' in errstr or + 'certificate revoked' in errstr or + 'certificate expired' in errstr or 'certificate verify failed' in errstr): return True diff --git a/koji/tasks.py b/koji/tasks.py index 7ceac2f..7bf7f3e 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -118,8 +118,7 @@ def parse_task_params(method, params): """ # check for new style - if (len(params) == 1 and isinstance(params[0], dict) - and '__method__' in params[0]): + if len(params) == 1 and isinstance(params[0], dict) and '__method__' in params[0]: ret = params[0].copy() del ret['__method__'] return ret diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index d267192..54f3b6c 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -83,7 +83,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): # path section are in form 'path%d' while order is important as some # paths can be mounted inside other mountpoints - path_sections = [p for p in cp.sections() if re.match('path\d+', p)] + path_sections = [p for p in cp.sections() if re.match(r'path\d+', p)] for section_name in sorted(path_sections, key=lambda x: int(x[4:])): try: self.config['paths'].append({ diff --git a/util/kojira b/util/kojira index 8ad82ce..92642d4 100755 --- a/util/kojira +++ b/util/kojira @@ -550,8 +550,7 @@ class RepoManager(object): stats = self.tagUseStats(entry['taginfo']['id']) # normalize use count - max_n = max([t.get('n_recent', 0) for t in self.needed_tags.values()] - or [1]) + max_n = max([t.get('n_recent', 0) for t in self.needed_tags.values()] or [1]) if max_n == 0: # no recent use or missing data max_n = 1 @@ -756,8 +755,7 @@ class RepoManager(object): if running_tasks >= self.options.max_repo_tasks: self.logger.info("Maximum number of repo tasks reached") return - elif (len(self.tasks) + len(self.other_tasks) - >= self.options.repo_tasks_limit): + elif len(self.tasks) + len(self.other_tasks) >= self.options.repo_tasks_limit: self.logger.info("Repo task limit reached") return tagname = tag['taginfo']['name'] diff --git a/vm/kojivmd b/vm/kojivmd index 3981783..a4da7d4 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -316,7 +316,9 @@ class DaemonXMLRPCServer(six.moves.xmlrpc_server.SimpleXMLRPCServer): except BaseException: # report exception back to server response = six.moves.xmlrpc_client.dumps( - six.moves.xmlrpc_client.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)) + six.moves.xmlrpc_client.Fault( + 1, "%s:%s" % + (sys.exc_info()[0], sys.exc_info()[1])) ) return response From 8c76bcde740d5a7aa84ebf55b42bf5aebd9f7f17 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 17/23] flake8: apply all rules after rebasing --- diff --git a/builder/kojid b/builder/kojid index 1c29e63..ed73891 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1948,7 +1948,8 @@ class WrapperRPMTask(BaseBuildTask): elif task['method'] == 'vmExec': self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release') values['win_info'] = {'platform': task_result['platform']} - elif task['method'] in ('createLiveCD', 'createAppliance', 'createImage', 'createLiveMedia'): + elif task['method'] in ('createLiveCD', 'createAppliance', 'createImage', + 'createLiveMedia'): self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release') else: # can't happen diff --git a/plugins/hub/runroot_hub.py b/plugins/hub/runroot_hub.py index 0a3e28a..726fc5f 100644 --- a/plugins/hub/runroot_hub.py +++ b/plugins/hub/runroot_hub.py @@ -9,11 +9,11 @@ import random import sys import koji -# XXX - have to import kojihub for make_task -sys.path.insert(0, '/usr/share/koji-hub/') -import kojihub from koji.context import context from koji.plugin import export +# XXX - have to import kojihub for make_task +sys.path.insert(0, '/usr/share/koji-hub/') +import kojihub # noqa: F402 __all__ = ('runroot',) diff --git a/plugins/hub/save_failed_tree.py b/plugins/hub/save_failed_tree.py index 0079223..8a1d644 100644 --- a/plugins/hub/save_failed_tree.py +++ b/plugins/hub/save_failed_tree.py @@ -3,12 +3,10 @@ from __future__ import absolute_import import sys import koji -sys.path.insert(0, '/usr/share/koji-hub/') -import kojihub from koji.context import context from koji.plugin import export - - +sys.path.insert(0, '/usr/share/koji-hub/') +import kojihub # noqa: F402 __all__ = ('saveFailedTree',) From 31006ed775771405eceb9f2b5c163562fe9593ab Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 18/23] add test-requirements.txt to install testing related modules by pip --- diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..7cba63d --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,6 @@ +flake8 +flake8-import-order +mock<=2.0.0 +requests-mock +coverage +nose From a3d3604416e40ad93d589bb3f9a452cad70704bf Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 19/23] flake8: update contribution guide for flake8 --- diff --git a/docs/source/writing_koji_code.rst b/docs/source/writing_koji_code.rst index 6e636b5..5911fdd 100644 --- a/docs/source/writing_koji_code.rst +++ b/docs/source/writing_koji_code.rst @@ -614,8 +614,8 @@ Here are some guidelines on producing preferable pull requests. - ``tests/test_cli/*`` - Check, that unit tests are not broken. Simply run ``make test`` in main - directory of your branch. For python3 compatible-code we have also ``make - test3`` target. + directory of your branch to check both python2/3 compatible-code. Or you can + also use ``make test2`` or ``make test3`` target for each of them. Note that the core development team for Koji is small, so it may take a few days for someone to reply to your request. @@ -657,3 +657,14 @@ on Koji. Unit tests are run automatically for any commit in master branch. We use Fedora's jenkins instance for that. Details are given here: :doc:`Unit tests in Fedora's Jenkins `. + +Code Style +========== + +We are using ``flake8`` to check the code style. Please refer to ``.flake8`` to +find the PEP8 and extra rules we are following/ignoring. + +You will need to install the packages below to run the check. + + * ``python-flake8`` + * ``python-flake8-import-order`` From b3b0c8d51ed57104bcad3267c77bac9f5f16fc5f Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 20/23] flake8: ignore F812 rule for PY2 --- diff --git a/.flake8 b/.flake8 index 1c4a908..3b7e0b3 100644 --- a/.flake8 +++ b/.flake8 @@ -5,6 +5,8 @@ ignore = E266, # do not assign a lambda expression, use a def E731, + # [PY2] list comprehension redefines `name` from line `N` + F812, # line break after binary operator W504 max_line_length = 99 From cf34706f04a62b0af2c8cc5843d2d2120736d03a Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 21/23] use Exception instead of BaseException for bare expection --- diff --git a/builder/kojid b/builder/kojid index ed73891..197fd54 100755 --- a/builder/kojid +++ b/builder/kojid @@ -167,7 +167,7 @@ def main(options, session): break except koji.RetryError: raise - except BaseException: + except Exception: # XXX - this is a little extreme # log the exception and continue logger.error(''.join(traceback.format_exception(*sys.exc_info()))) @@ -488,7 +488,7 @@ class BuildRoot(object): fd.close() fd = open(fpath, 'rb') logs[fname] = (fd, stat_info.st_ino, stat_info.st_size or size, fpath) - except BaseException: + except Exception: self.logger.error("Error reading mock log: %s", fpath) self.logger.error(''.join(traceback.format_exception(*sys.exc_info()))) continue @@ -533,7 +533,7 @@ class BuildRoot(object): os.setregid(gid, gid) os.setreuid(uid, uid) os.execvp(cmd[0], cmd) - except BaseException: + except Exception: # diediedie print("Failed to exec mock") print(''.join(traceback.format_exception(*sys.exc_info()))) @@ -801,7 +801,7 @@ class BuildRoot(object): with koji.openRemoteFile(repomdpath, **opts) as fo: try: repodata = repoMDObject.RepoMD('ourrepo', fo) - except BaseException: + except Exception: raise koji.BuildError("Unable to parse repomd.xml file for %s" % os.path.join(repodir, self.br_arch)) data = repodata.getData('origin') @@ -1061,7 +1061,7 @@ class BuildTask(BaseTaskHandler): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not self.opts.get('scratch'): # scratch builds do not get imported self.session.host.failBuild(self.id, build_id) @@ -1547,7 +1547,7 @@ class MavenTask(MultiPlatformTask): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not self.opts.get('scratch'): # scratch builds do not get imported self.session.host.failBuild(self.id, self.build_id) @@ -2067,7 +2067,7 @@ class WrapperRPMTask(BaseBuildTask): buildroot.build(srpm) except (SystemExit, ServerExit, KeyboardInterrupt): raise - except BaseException: + except Exception: if self.new_build_id: self.session.host.failBuild(self.id, self.new_build_id) raise @@ -2111,7 +2111,7 @@ class WrapperRPMTask(BaseBuildTask): self.uploadFile(os.path.join(resultdir, rpm_fn)) except (SystemExit, ServerExit, KeyboardInterrupt): raise - except BaseException: + except Exception: if self.new_build_id: self.session.host.failBuild(self.id, self.new_build_id) raise @@ -2140,7 +2140,7 @@ class WrapperRPMTask(BaseBuildTask): {'noarch': rellogs}) except (SystemExit, ServerExit, KeyboardInterrupt): raise - except BaseException: + except Exception: self.session.host.failBuild(self.id, self.new_build_id) raise if not opts.get('skip_tag'): @@ -2540,7 +2540,7 @@ class BuildBaseImageTask(BuildImageTask): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not opts.get('scratch'): # scratch builds do not get imported if bld_info: @@ -2631,7 +2631,7 @@ class BuildApplianceTask(BuildImageTask): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not opts.get('scratch'): # scratch builds do not get imported if bld_info: @@ -2718,7 +2718,7 @@ class BuildLiveCDTask(BuildImageTask): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not opts.get('scratch'): # scratch builds do not get imported if bld_info: @@ -2857,7 +2857,7 @@ class BuildLiveMediaTask(BuildImageTask): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not opts.get('scratch'): # scratch builds do not get imported if bld_info: @@ -4656,7 +4656,7 @@ class BuildIndirectionImageTask(OzImageTask): return self._do_indirection(opts, base_factory_image, utility_factory_image, indirection_template, tlog, ozlog, fhandler, bld_info, target_info, bd) - except BaseException: + except Exception: if not opts.get('scratch'): # scratch builds do not get imported if bld_info: @@ -5129,7 +5129,7 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r result = None try: result = self.session.getTaskResult(task_id) - except BaseException: + except Exception: excClass, result = sys.exc_info()[:2] if hasattr(result, 'faultString'): result = result.faultString diff --git a/cli/koji b/cli/koji index c450074..08bd6d5 100755 --- a/cli/koji +++ b/cli/koji @@ -337,7 +337,7 @@ if __name__ == "__main__": rv = 0 except (KeyboardInterrupt, SystemExit): rv = 1 - except BaseException: + except Exception: if options.debug: raise else: @@ -346,6 +346,6 @@ if __name__ == "__main__": logger.error("%s: %s" % (exctype.__name__, value)) try: session.logout() - except BaseException: + except Exception: pass sys.exit(rv) diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index df7c95b..11353be 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -3056,7 +3056,7 @@ def anon_handle_list_builds(goptions, session, args): dt = dateutil.parser.parse(val) ts = time.mktime(dt.timetuple()) setattr(options, opt, ts) - except BaseException: + except Exception: parser.error(_("Invalid time specification: %s") % val) if options.before: opts['completeBefore'] = getattr(options, 'before') @@ -4450,7 +4450,7 @@ def anon_handle_list_history(goptions, session, args): dt = dateutil.parser.parse(val) ts = time.mktime(dt.timetuple()) setattr(options, opt, ts) - except BaseException: + except Exception: parser.error(_("Invalid time specification: %s") % val) for opt in ('package', 'tag', 'build', 'editor', 'user', 'permission', 'cg', 'external_repo', 'build_target', 'group', 'before', diff --git a/hub/kojihub.py b/hub/kojihub.py index b350cca..46db0c5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2672,7 +2672,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa relpath = os.path.relpath(srcdir, dest_parent) try: os.symlink(relpath, destlink) - except BaseException: + except Exception: log_error('Error linking %s to %s' % (destlink, relpath)) for artifact_dir, artifacts in six.iteritems(artifact_dirs): _write_maven_repo_metadata(artifact_dir, artifacts) @@ -4989,7 +4989,7 @@ def list_task_output(taskID, stat=False, all_volumes=False, strict=False): # raise error if task doesn't exist try: Task(taskID).getInfo(strict=True) - except BaseException: + except Exception: raise koji.GenericError("Task doesn't exist") if stat or all_volumes: @@ -7375,10 +7375,10 @@ def check_rpm_sig(an_rpm, sigkey, sighdr): ts.setVSFlags(0) # full verify with open(temp, 'rb') as fo: hdr = ts.hdrFromFdno(fo.fileno()) - except BaseException: + except Exception: try: os.unlink(temp) - except BaseException: + except Exception: pass raise raw_key = koji.get_header_field(hdr, 'siggpg') diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 8f2c202..d1aa728 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -242,7 +242,7 @@ class ModXMLRPCRequestHandler(object): except Fault as fault: self.traceback = True response = dumps(fault, marshaller=Marshaller) - except BaseException: + except Exception: self.traceback = True # report exception back to server e_class, e = sys.exc_info()[:2] diff --git a/koji/__init__.py b/koji/__init__.py index e4b01a1..4a4335f 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -303,10 +303,10 @@ class GenericError(Exception): def __str__(self): try: return str(self.args[0]['args'][0]) - except BaseException: + except Exception: try: return str(self.args[0]) - except BaseException: + except Exception: return str(self.__dict__) # END kojikamid dup # @@ -1730,7 +1730,7 @@ def format_exc_plus(): # COULD cause any exception, so we MUST catch any...: try: rv += "%s\n" % value - except BaseException: + except Exception: rv += "\n" return rv @@ -2273,7 +2273,7 @@ class VirtualMethod(object): self.__session._apidoc = dict( [(f["name"], f) for f in self.__func("_listapi", [], {})] ) - except BaseException: + except Exception: self.__session._apidoc = {} funcdoc = self.__session._apidoc.get(self.__name) @@ -2659,7 +2659,7 @@ class ClientSession(object): if self.__dict__: try: self.logout() - except BaseException: + except Exception: pass def callMethod(self, name, *args, **opts): @@ -3303,7 +3303,7 @@ class DBHandler(logging.Handler): # self.cnx.commit() # XXX - committing here is most likely wrong, but we need to set commit_pending or # something...and this is really the wrong place for that - except BaseException: + except Exception: self.handleError(record) diff --git a/koji/arch.py b/koji/arch.py index d2dc87d..ac4d5a0 100644 --- a/koji/arch.py +++ b/koji/arch.py @@ -238,7 +238,7 @@ def _try_read_cpuinfo(): mounted). """ try: return open("/proc/cpuinfo", "r") - except BaseException: + except Exception: return [] @@ -248,7 +248,7 @@ def _parse_auxv(): # In case we can't open and read /proc/self/auxv, just return try: data = open("/proc/self/auxv", "rb").read() - except BaseException: + except Exception: return # Define values from /usr/include/elf.h @@ -323,7 +323,7 @@ def getCanonPPCArch(arch): try: if platform.startswith("power") and int(platform[5:].rstrip('+')) >= 7: return "ppc64p7" - except BaseException: + except Exception: pass if machine is None: @@ -388,7 +388,7 @@ def getCanonArch(skipRpmPlatform=0): f.close() (arch, vendor, opersys) = line.split("-", 2) return arch - except BaseException: + except Exception: pass arch = os.uname()[4] diff --git a/koji/auth.py b/koji/auth.py index 1a3eef6..bc73c02 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -95,7 +95,7 @@ class Session(object): raise koji.AuthError('%s not specified in session args' % field) try: callnum = args['callnum'][0] - except BaseException: + except Exception: callnum = None # lookup the session c = context.cnx.cursor() diff --git a/koji/daemon.py b/koji/daemon.py index ad425d9..1a75857 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -139,7 +139,7 @@ def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, a if env: environ.update(env) os.execvpe(path, args, environ) - except BaseException: + except Exception: msg = ''.join(traceback.format_exception(*sys.exc_info())) if fd: try: @@ -148,7 +148,7 @@ def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, a else: os.write(fd, msg) os.close(fd) - except BaseException: + except Exception: pass print(msg) os._exit(1) @@ -167,7 +167,7 @@ def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, a except IOError: # will happen if the forked process has not created the logfile yet continue - except BaseException: + except Exception: print('Error reading log file: %s' % outfile) print(''.join(traceback.format_exception(*sys.exc_info()))) @@ -1163,7 +1163,7 @@ class TaskManager(object): try: self.session.logoutChild(session_id) del self.subsessions[task_id] - except BaseException: + except Exception: # not much we can do about it pass if wait: @@ -1266,7 +1266,7 @@ class TaskManager(object): valid_host = handler.checkHost(self.hostdata) except (SystemExit, KeyboardInterrupt): raise - except BaseException: + except Exception: valid_host = False self.logger.warn('Error during host check') self.logger.warn(''.join(traceback.format_exception(*sys.exc_info()))) @@ -1350,7 +1350,7 @@ class TaskManager(object): # freeing this task will allow the pending restart to take effect self.session.host.freeTasks([handler.id]) return - except BaseException: + except Exception: tb = ''.join(traceback.format_exception(*sys.exc_info())) self.logger.warn("TRACEBACK: %s" % tb) # report exception back to server diff --git a/koji/plugin.py b/koji/plugin.py index 1342825..4c98bc6 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -199,7 +199,7 @@ def run_callbacks(cbtype, *args, **kws): cb_args, cb_kwargs = _fix_cb_args(func, args, kws, cache) try: func(cbtype, *cb_args, **cb_kwargs) - except BaseException: + except Exception: msg = 'Error running %s callback from %s' % (cbtype, func.__module__) if getattr(func, 'failure_is_an_option', False): logging.getLogger('koji.plugin').warn(msg, exc_info=True) diff --git a/koji/rpmdiff.py b/koji/rpmdiff.py index 6aac46a..fdd34af 100644 --- a/koji/rpmdiff.py +++ b/koji/rpmdiff.py @@ -71,7 +71,7 @@ class Rpmdiff: except AttributeError: try: PREREQ_FLAG = rpm.RPMSENSE_PREREQ - except BaseException: + except Exception: # (proyvind): This seems ugly, but then again so does # this whole check as well. PREREQ_FLAG = False diff --git a/koji/tasks.py b/koji/tasks.py index 7bf7f3e..4fe7273 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -79,7 +79,7 @@ def safe_rmtree(path, unmount=False, strict=True): logger.debug("Removing: %s" % path) try: os.remove(path) - except BaseException: + except Exception: if strict: raise else: diff --git a/util/koji-gc b/util/koji-gc index 74fd0b8..eafc13c 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -451,7 +451,7 @@ Build: %%(name)s-%%(version)s-%%(release)s s.login(options.smtp_user, options.smtp_pass) s.sendmail(msg['From'], msg['To'], msg.as_string()) s.quit() - except BaseException: + except Exception: print("FAILED: Sending warning notice to %s" % msg['To']) @@ -1010,7 +1010,7 @@ if __name__ == "__main__": if options.exit_on_lock: try: session.logout() - except BaseException: + except Exception: pass sys.exit(1) os.close(lock_fd) @@ -1041,7 +1041,7 @@ if __name__ == "__main__": # print("%s: %s" % (exctype, value)) try: session.logout() - except BaseException: + except Exception: pass if not options.skip_main: sys.exit(rv) diff --git a/util/koji-shadow b/util/koji-shadow index 55490e4..52d2e0c 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -454,7 +454,7 @@ class TrackedBuild(object): # XXX - Move SCM class out of kojid and use it to check for scm url if src.startswith('cvs:'): return src - except BaseException: + except Exception: pass # otherwise fail return None @@ -1286,7 +1286,7 @@ def main(args): filename = options.logfile try: logfile = os.open(filename, os.O_CREAT | os.O_RDWR | os.O_APPEND, 0o777) - except BaseException: + except Exception: logfile = None if logfile is not None: log("logging to %s" % filename) @@ -1341,6 +1341,6 @@ if __name__ == "__main__": # log ("%s: %s" % (exctype, value)) try: session.logout() - except BaseException: + except Exception: pass sys.exit(rv) diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 48da704..d324777 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -76,7 +76,7 @@ def clean_scratch_tasks(cursor, vacuum, test, age): if opts['scratch']: cursor.execute("INSERT INTO temp_scratch_tasks VALUES (%s)", (task_id,)) ids.append(task_id) - except BaseException: + except Exception: continue parents = ids diff --git a/util/kojira b/util/kojira index 92642d4..d963c9e 100755 --- a/util/kojira +++ b/util/kojira @@ -410,7 +410,7 @@ class RepoManager(object): while True: self.checkCurrentRepos() time.sleep(self.options.sleeptime) - except BaseException: + except Exception: self.logger.exception('Error in currency checker thread') raise finally: @@ -425,7 +425,7 @@ class RepoManager(object): while True: self.regenRepos() time.sleep(self.options.sleeptime) - except BaseException: + except Exception: self.logger.exception('Error in regen thread') raise finally: @@ -856,7 +856,7 @@ def main(options, session): except SystemExit: logger.warn("Shutting down") break - except BaseException: + except Exception: # log the exception and continue logger.error(''.join(traceback.format_exception(*sys.exc_info()))) try: @@ -1005,7 +1005,7 @@ if __name__ == "__main__": try: logfile = open(options.logfile, "w") logfile.close() - except BaseException: + except Exception: sys.stderr.write("Cannot create logfile: %s\n" % options.logfile) sys.exit(1) if not os.access(options.logfile, os.W_OK): diff --git a/vm/kojikamid.py b/vm/kojikamid.py index d03d9ad..ad41fed 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -699,7 +699,7 @@ def stream_logs(server, handler, builds): try: fd = open(log, 'r') logs[log] = (relpath, fd) - except BaseException: + except Exception: log_local('Error opening %s' % log) continue else: @@ -713,7 +713,7 @@ def stream_logs(server, handler, builds): del contents try: server.uploadDirect(relpath, offset, size, digest, data) - except BaseException: + except Exception: log_local('error uploading %s' % relpath) time.sleep(1) @@ -729,14 +729,14 @@ def fail(server, handler): logfd.flush() upload_file(server, os.path.dirname(logfile), os.path.basename(logfile)) - except BaseException: + except Exception: log_local('error calling upload_file()') while True: try: # this is the very last thing we do, keep trying as long as we can server.failTask(tb) break - except BaseException: + except Exception: log_local('error calling server.failTask()') sys.exit(1) @@ -806,7 +806,7 @@ def main(): results['logs'].append(os.path.basename(logfile)) server.closeTask(results) - except BaseException: + except Exception: fail(server, handler) sys.exit(0) diff --git a/vm/kojivmd b/vm/kojivmd index a4da7d4..4677f54 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -241,7 +241,7 @@ def main(options, session): break except koji.RetryError: raise - except BaseException: + except Exception: # XXX - this is a little extreme # log the exception and continue logger.error('Error in main loop', exc_info=True) @@ -295,7 +295,7 @@ class DaemonXMLRPCServer(six.moves.xmlrpc_server.SimpleXMLRPCServer): self.close_request(conn) except socket.timeout: pass - except BaseException: + except Exception: self.logger.error('Error handling requests', exc_info=True) if sys.version_info[:2] <= (2, 4): @@ -313,7 +313,7 @@ class DaemonXMLRPCServer(six.moves.xmlrpc_server.SimpleXMLRPCServer): methodresponse=1, allow_none=True) except six.moves.xmlrpc_client.Fault as fault: response = six.moves.xmlrpc_client.dumps(fault) - except BaseException: + except Exception: # report exception back to server response = six.moves.xmlrpc_client.dumps( six.moves.xmlrpc_client.Fault( @@ -415,7 +415,7 @@ class WinBuildTask(MultiPlatformTask): except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise - except BaseException: + except Exception: if not opts.get('scratch'): # scratch builds do not get imported self.session.host.failBuild(self.id, build_id) @@ -1056,7 +1056,7 @@ class VMTaskManager(TaskManager): if os.path.isfile(disk): os.unlink(disk) self.logger.debug('Removed disk file %s for VM %s', disk, vm_name) - except BaseException: + except Exception: self.logger.error('Error removing disk file %s for VM %s', disk, vm_name, exc_info=True) return False diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index f62e96c..f7bbcc6 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -714,7 +714,7 @@ def taskinfo(environ, taskID): if task['state'] in (koji.TASK_STATES['CLOSED'], koji.TASK_STATES['FAILED']): try: result = server.getTaskResult(task['id']) - except BaseException: + except Exception: excClass, exc = sys.exc_info()[:2] values['result'] = exc values['excClass'] = excClass @@ -748,7 +748,7 @@ def taskinfo(environ, taskID): try: values['params_parsed'] = _genHTML(environ, 'taskinfo_params.chtml') - except BaseException: + except Exception: values['params_parsed'] = None return _genHTML(environ, 'taskinfo.chtml') @@ -2458,7 +2458,7 @@ def search(environ, start=None, order=None): if match == 'regexp': try: re.compile(terms) - except BaseException: + except Exception: values['error'] = 'Invalid regular expression' return _genHTML(environ, 'search.chtml') diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 97477ec..44167bd 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -207,7 +207,7 @@ class Dispatcher(object): args = inspect.getargspec(val) if not args[0] or args[0][0] != 'environ': continue - except BaseException: + except Exception: tb_str = ''.join(traceback.format_exception(*sys.exc_info())) self.logger.error(tb_str) self.handler_index[name] = val diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index e42a6e5..ada7f02 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -45,7 +45,7 @@ class NoSuchException(Exception): try: # pyOpenSSL might not be around from OpenSSL.SSL import Error as SSL_Error -except BaseException: +except Exception: SSL_Error = NoSuchException From d21083a11be947a6a96944d541476d41c50c7f2c Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 22/23] still use BaseException for logging purpose --- diff --git a/builder/kojid b/builder/kojid index 197fd54..a9c8cf1 100755 --- a/builder/kojid +++ b/builder/kojid @@ -533,7 +533,7 @@ class BuildRoot(object): os.setregid(gid, gid) os.setreuid(uid, uid) os.execvp(cmd[0], cmd) - except Exception: + except BaseException: # diediedie print("Failed to exec mock") print(''.join(traceback.format_exception(*sys.exc_info()))) diff --git a/koji/daemon.py b/koji/daemon.py index 1a75857..0167c5b 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -139,7 +139,7 @@ def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, a if env: environ.update(env) os.execvpe(path, args, environ) - except Exception: + except BaseException: msg = ''.join(traceback.format_exception(*sys.exc_info())) if fd: try: diff --git a/util/kojira b/util/kojira index d963c9e..f25c1af 100755 --- a/util/kojira +++ b/util/kojira @@ -322,7 +322,7 @@ class RepoManager(object): try: rmtree(path) status = 0 - except Exception: + except BaseException: logger.error(''.join(traceback.format_exception(*sys.exc_info()))) logging.shutdown() finally: From 20ee6081e0faa93e930154a99bf914f495ba687c Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Mar 03 2020 13:38:22 +0000 Subject: [PATCH 23/23] flake8: apply rules for koji-sidetag-cleanup --- diff --git a/.flake8 b/.flake8 index 3b7e0b3..2051214 100644 --- a/.flake8 +++ b/.flake8 @@ -27,6 +27,7 @@ filename = ./util/koji-gc, ./util/koji-shadow, ./util/koji-sweep-db, + ./util/koji-sidetag-cleanup, ./vm/kojivmd application_import_names = koji,koji_cli,kojihub,kojiweb,__main__ diff --git a/util/koji-sidetag-cleanup b/util/koji-sidetag-cleanup index 9cb8691..2d4f026 100644 --- a/util/koji-sidetag-cleanup +++ b/util/koji-sidetag-cleanup @@ -53,20 +53,17 @@ def get_options(): help=_("show xmlrpc debug output")) parser.add_option("-t", "--test", action="store_true", help=_("test mode, no tag is deleted")) - - parser.add_option("--no-empty", action="store_false", dest="clean_empty", - default=True, help=_("don't run emptiness check")) - parser.add_option("--empty-delay", action="store", metavar="DAYS", - default=1, type=int, + parser.add_option("--no-empty", action="store_false", dest="clean_empty", default=True, + help=_("don't run emptiness check")) + parser.add_option("--empty-delay", action="store", metavar="DAYS", default=1, type=int, help=_("delete empty tags older than DAYS")) - parser.add_option("--no-old", action="store_false", dest="clean_old", - default=True, help=_("don't run old check")) - parser.add_option("--old-delay", action="store", metavar="DAYS", - default=30, type=int, + parser.add_option("--no-old", action="store_false", dest="clean_old", default=True, + help=_("don't run old check")) + parser.add_option("--old-delay", action="store", metavar="DAYS", default=30, type=int, help=_("delete older tags than timestamp")) parser.add_option("--ignore-tags", metavar="PATTERN", action="append", help=_("Ignore tags matching PATTERN when pruning")) - #parse once to get the config file + # parse once to get the config file (options, args) = parser.parse_args() defaults = parser.get_default_values() @@ -115,7 +112,7 @@ def get_options(): setattr(defaults, name, config.getboolean(*alias)) else: setattr(defaults, name, config.get(*alias)) - #parse again with defaults + # parse again with defaults (options, args) = parser.parse_args(values=defaults) options.config = config @@ -139,25 +136,29 @@ def ensure_connection(session): except xmlrpc.client.ProtocolError: error(_("Unable to connect to server")) if ret != koji.API_VERSION: - warn(_("The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) + warn(_("The server is at API version %d and the client is at %d" % + (ret, koji.API_VERSION))) def activate_session(session): """Test and login the session is applicable""" global options if options.noauth: - #skip authentication + # skip authentication pass elif options.cert is not None and os.path.isfile(options.cert): # authenticate using SSL client cert session.ssl_login(options.cert, None, options.serverca, proxyuser=options.runas) elif options.user: - #authenticate using user/password + # authenticate using user/password session.login() elif options.keytab and options.principal: try: if options.keytab and options.principal: - session.gssapi_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) + session.gssapi_login( + principal=options.principal, + keytab=options.keytab, + proxyuser=options.runas) else: session.gssapi_login(proxyuser=options.runas) except Exception as e: @@ -250,12 +251,14 @@ def clean_old(tags): delete_tags(deleted) return passed + def main(args): activate_session(session) sidetags = get_all() sidetags = clean_empty(sidetags) sidetags = clean_old(sidetags) + if __name__ == "__main__": options, args = get_options() session_opts = koji.grab_session_options(options)