From 8168da12cd473388978a8715c66157e5954a0230 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 04 2016 17:44:35 +0000 Subject: [PATCH 1/9] add rc archivetype --- diff --git a/docs/schema.sql b/docs/schema.sql index 1968cd1..00f04a6 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -821,6 +821,7 @@ insert into archivetypes (name, description, extensions) values ('dot', 'DOT gra insert into archivetypes (name, description, extensions) values ('groovy', 'Groovy script file', 'groovy gvy'); insert into archivetypes (name, description, extensions) values ('batch', 'Batch file', 'bat'); insert into archivetypes (name, description, extensions) values ('shell', 'Shell script', 'sh'); +insert into archivetypes (name, description, extensions) values ('rc', 'Resource file', 'rc'); -- Do we want to enforce a constraint that a build can only generate one From de92c542dcc763446ccd0307bf34d5efb7b41864 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 04 2016 20:15:00 +0000 Subject: [PATCH 2/9] clarify error message for noarch rpmdiff mismatches see https://fedorahosted.org/koji/ticket/325 --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 886d1a0..c84db08 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -7752,8 +7752,9 @@ def rpmdiff(basepath, rpmlist): status = proc.wait() if os.WIFSIGNALED(status) or \ (os.WEXITSTATUS(status) != 0): - raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \ - (os.path.basename(first_rpm), output) + raise koji.BuildError( + 'The following noarch package built differently on different architectures: %s\n' + 'rpmdiff output was:\n%s' % (os.path.basename(first_rpm), output)) def importImageInternal(task_id, build_id, imgdata): """ From 7e56b133a099cb8e9c7431eb6d4e6108f05e1032 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2016 22:56:40 +0000 Subject: [PATCH 3/9] handle Unexpected EOF exceptions on reads --- diff --git a/koji/ssl/SSLConnection.py b/koji/ssl/SSLConnection.py index fae25e9..e80bf44 100644 --- a/koji/ssl/SSLConnection.py +++ b/koji/ssl/SSLConnection.py @@ -142,6 +142,10 @@ class SSLConnection: return None except SSL.WantReadError: time.sleep(0.2) + except SSL.SysCallError, e: + if e.args == (-1, 'Unexpected EOF'): + break + raise return None class PlgFileObject(socket._fileobject): From 33cf1ab6f2c71bb7481b324d6b141fa10bba928a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2016 22:56:40 +0000 Subject: [PATCH 4/9] avoid masking exceptions in retry code --- diff --git a/koji/__init__.py b/koji/__init__.py index ebdd4b8..d43b8ef 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -58,7 +58,6 @@ import xmlrpclib import xml.sax import xml.sax.handler from xmlrpclib import loads, dumps, Fault -import OpenSSL import zipfile def _(args): @@ -1959,34 +1958,11 @@ class ClientSession(object): raise except Exception, e: self._close_connection() - if isinstance(e, OpenSSL.SSL.Error): - # pyOpenSSL doesn't use different exception - # subclasses, we have to actually parse the args - for arg in e.args: - # First, check to see if 'arg' is iterable because - # it can be anything.. - try: - iter(arg) - except TypeError: - continue - - # We do all this so that we can detect cert expiry - # so we can avoid retrying those over and over. - for items in arg: - try: - iter(items) - except TypeError: - continue - - if len(items) != 3: - continue - - _, _, ssl_reason = items - - if ('certificate revoked' in ssl_reason or - 'certificate expired' in ssl_reason): - # There's no point in retrying for this - raise + + if ssl.SSLCommon.is_cert_error(e): + # There's no point in retrying for this + raise + if not self.logged_in: #in the past, non-logged-in sessions did not retry. For compatibility purposes #this behavior is governed by the anon_retry opt. diff --git a/koji/ssl/SSLCommon.py b/koji/ssl/SSLCommon.py index 56efc05..345d4ea 100644 --- a/koji/ssl/SSLCommon.py +++ b/koji/ssl/SSLCommon.py @@ -29,6 +29,43 @@ def our_verify(connection, x509, errNum, errDepth, preverifyOK): return preverifyOK +def is_cert_error(e): + """Determine if an OpenSSL error is due to a bad cert""" + + if not isinstance(e, SSL.Error): + return False + + # pyOpenSSL doesn't use different exception + # subclasses, we have to actually parse the args + for arg in e.args: + # First, check to see if 'arg' is iterable because + # it can be anything.. + try: + iter(arg) + except TypeError: + continue + + # We do all this so that we can detect cert expiry + # so we can avoid retrying those over and over. + for items in arg: + try: + iter(items) + except TypeError: + continue + + if len(items) != 3: + continue + + _, _, ssl_reason = items + + if ('certificate revoked' in ssl_reason or + 'certificate expired' in ssl_reason): + return True + + #otherwise + return False + + def CreateSSLContext(certs): key_and_cert = certs['key_and_cert'] peer_ca_cert = certs['peer_ca_cert'] From a6c218df947c7a73538339b373a05c01426d48ef Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 11 2016 19:30:55 +0000 Subject: [PATCH 5/9] avoid traceback when cli fails to parse task parameters also, fix parameter parsing for createLiveCD and createAppliance --- diff --git a/cli/koji b/cli/koji index f383c6f..aff0aaf 100755 --- a/cli/koji +++ b/cli/koji @@ -4224,7 +4224,18 @@ def _handleOpts(lines, opts, prefix=''): lines.append("%sOptions:" % prefix) _handleMap(lines, opts, prefix) + def _parseTaskParams(session, method, task_id): + try: + return _do_parseTaskParams(session, method, task_id) + except Exception: + if logger.isEnabledFor(logging.DEBUG): + tb_str = ''.join(traceback.format_exception(*sys.exc_info())) + logger.debug(tb_str) + return ['Unable to parse task parameters'] + + +def _do_parseTaskParams(session, method, task_id): """Parse the return of getTaskRequest()""" params = session.getTaskRequest(task_id) @@ -4301,11 +4312,10 @@ def _parseTaskParams(session, method, task_id): if len(params) > 2: _handleOpts(lines, params[2]) elif method in ('createLiveCD', 'createAppliance'): - lines.append("Arch: %s" % params[0]) - lines.append("Build Target: %s" % params[1]) - lines.append("Kickstart File: %s" % params[2]) - if len(params) > 3: - _handleOpts(lines, params[3]) + lines.append("Arch: %s" % params[3]) + lines.append("Kickstart File: %s" % params[7]) + if len(params) > 8: + _handleOpts(lines, params[8]) elif method == 'newRepo': tag = session.getTag(params[0]) lines.append("Tag: %s" % tag['name']) From a621081b4e7af81caa27dab966f83f1fdbabe8a4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 15 2016 17:01:51 +0000 Subject: [PATCH 6/9] wsdl archivetype --- diff --git a/docs/schema.sql b/docs/schema.sql index 00f04a6..18402d8 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -822,6 +822,7 @@ insert into archivetypes (name, description, extensions) values ('groovy', 'Groo insert into archivetypes (name, description, extensions) values ('batch', 'Batch file', 'bat'); insert into archivetypes (name, description, extensions) values ('shell', 'Shell script', 'sh'); insert into archivetypes (name, description, extensions) values ('rc', 'Resource file', 'rc'); +insert into archivetypes (name, description, extensions) values ('wsdl', 'Web Services Description Language', 'wsdl'); -- Do we want to enforce a constraint that a build can only generate one From 4365699e849be87faae528cd26baa3a1acd3b26c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 18 2016 21:22:59 +0000 Subject: [PATCH 7/9] add-pkg: use multicall and ignore existing packages --- diff --git a/cli/koji b/cli/koji index aff0aaf..581138d 100755 --- a/cli/koji +++ b/cli/koji @@ -766,19 +766,23 @@ def handle_add_pkg(options, session, args): print "No such tag: %s" % tag sys.exit(1) pkglist = dict([(p['package_name'], p['package_id']) for p in session.listPackages(tagID=dsttag['id'])]) - ret = 0 + to_add = [] for package in args[1:]: package_id = pkglist.get(package, None) if not package_id is None: print "Package %s already exists in tag %s" % (package, tag) - ret = 1 - if ret: - return ret + continue + to_add.append(package) if options.extra_arches: opts['extra_arches'] = ' '.join(options.extra_arches.replace(',',' ').split()) - for package in args[1:]: - #really should implement multicall... - session.packageListAdd(tag,package,options.owner,**opts) + + # add the packages + print "Adding %i packages to tag %s" % (len(to_add), dsttag['name']) + session.multicall = True + for package in to_add: + session.packageListAdd(tag, package, options.owner, **opts) + session.multiCall(strict=True) + def handle_block_pkg(options, session, args): "[admin] Block a package in the listing for tag" From 393dba2fa97ad2aa34015daae1163b4c13daaff6 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 18 2016 21:34:09 +0000 Subject: [PATCH 8/9] cli: handle rpms in download-build --- diff --git a/cli/koji b/cli/koji index 581138d..d089dc0 100755 --- a/cli/koji +++ b/cli/koji @@ -6212,6 +6212,7 @@ def anon_handle_download_build(options, session, args): parser.add_option("--latestfrom", dest="latestfrom", help=_("Download the latest build from this tag")) parser.add_option("--debuginfo", action="store_true", help=_("Also download -debuginfo rpms")) parser.add_option("--task-id", action="store_true", help=_("Interperet id as a task id")) + parser.add_option("--rpm", action="store_true", help=_("Download the given rpm")) parser.add_option("--key", help=_("Download rpms signed with the given key")) parser.add_option("--topurl", metavar="URL", default=options.topurl, help=_("URL under which Koji files are accessible")) @@ -6251,6 +6252,12 @@ def anon_handle_download_build(options, session, args): print "%s has no builds of %s" % (suboptions.latestfrom, build) return 1 info = builds[0] + elif suboptions.rpm: + rpminfo = session.getRPM(build) + if rpminfo is None: + print "No such rpm: %s" % build + return 1 + info = session.getBuild(rpminfo['build_id']) else: info = session.getBuild(build) @@ -6292,7 +6299,10 @@ def anon_handle_download_build(options, session, args): arches = suboptions.arches if len(arches) == 0: arches = None - rpms = session.listRPMs(buildID=info['id'], arches=arches) + if suboptions.rpm: + rpms = [rpminfo] + else: + rpms = session.listRPMs(buildID=info['id'], arches=arches) if not rpms: if arches: print "No %s packages available for %s" % (" or ".join(arches), koji.buildLabel(info)) From 709558eaa96762ab70f6d8d268f3521832abcc73 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 18 2016 21:34:15 +0000 Subject: [PATCH 9/9] cli: if given an rpm name without --rpm, download the containing build --- diff --git a/cli/koji b/cli/koji index d089dc0..e0eef14 100755 --- a/cli/koji +++ b/cli/koji @@ -6206,7 +6206,7 @@ def anon_handle_download_build(options, session, args): usage = _("usage: %prog download-build [options] ") usage += _("\n(Specify the --help global option for a list of other help options)") parser = OptionParser(usage=usage) - parser.add_option("--arch", dest="arches", metavar="ARCH", action="append", default=[], + parser.add_option("--arch", "-a", dest="arches", metavar="ARCH", action="append", default=[], help=_("Only download packages for this arch (may be used multiple times)")) parser.add_option("--type", help=_("Download archives of the given type, rather than rpms (maven, win, or image)")) parser.add_option("--latestfrom", dest="latestfrom", help=_("Download the latest build from this tag")) @@ -6259,6 +6259,13 @@ def anon_handle_download_build(options, session, args): return 1 info = session.getBuild(rpminfo['build_id']) else: + # if we're given an rpm name without --rpm, download the containing build + try: + nvra = koji.parse_NVRA(build) + rpminfo = session.getRPM(build) + build = rpminfo['build_id'] + except Exception: + pass info = session.getBuild(build) if info is None: