From 040b4b62e8aea84a1915d5e7d7253345af7db608 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:06:45 +0000 Subject: [PATCH 1/17] python-modernize -f libmodernize.fixes.fix_zip . -w --- diff --git a/hub/kojihub.py b/hub/kojihub.py index dedcbc6..94c1f7f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -22,6 +22,7 @@ # Mike Bonnet # Cristian Balint +from __future__ import absolute_import import base64 import calendar import urlparse @@ -58,6 +59,7 @@ import zipfile import koji.xmlrpcplus from koji.context import context +from six.moves import zip try: import json @@ -1307,7 +1309,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late else: raise koji.GenericError('invalid arch option: %s' % arch) - fields, aliases = zip(*fields) + fields, aliases = list(zip(*fields)) query = QueryProcessor(tables=tables, joins=joins, clauses=clauses, columns=fields, aliases=aliases, values=data, transform=_fix_rpm_row) @@ -2550,7 +2552,7 @@ def repo_references(repo_id): 'host_id': 'host_id', 'create_event': 'create_event', 'state': 'state'} - fields, aliases = zip(*fields.items()) + fields, aliases = list(zip(*fields.items())) values = {'repo_id': repo_id} clauses = ['repo_id=%(repo_id)s', 'retire_event IS NULL'] query = QueryProcessor(columns=fields, aliases=aliases, tables=['standard_buildroot'], @@ -2579,7 +2581,7 @@ def get_active_repos(): ('repo.dist', 'dist'), ('tag.name', 'tag_name'), ) - fields, aliases = zip(*fields) + fields, aliases = list(zip(*fields)) values = {'st_deleted': koji.REPO_DELETED} joins = ['tag ON repo.tag_id=tag.id', 'events ON repo.create_event = events.id'] clauses = ['repo.state != %(st_deleted)s'] @@ -2984,7 +2986,7 @@ def get_tag(tagInfo, strict=False, event=None): raise koji.GenericError('invalid type for tagInfo: %s' % type(tagInfo)) data = {'tagInfo': tagInfo} - fields, aliases = zip(*fields.items()) + fields, aliases = list(zip(*fields.items())) query = QueryProcessor(columns=fields, aliases=aliases, tables=tables, joins=joins, clauses=clauses, values=data) result = query.executeOne() @@ -3529,7 +3531,7 @@ def get_build(buildInfo, strict=False): ('users.id', 'owner_id'), ('users.name', 'owner_name'), ('build.source', 'source'), ('build.extra', 'extra')) - fields, aliases = zip(*fields) + fields, aliases = list(zip(*fields)) joins = ['events ON build.create_event = events.id', 'package on build.pkg_id = package.id', 'volume on build.volume_id = volume.id', @@ -3793,7 +3795,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID else: raise koji.GenericError('invalid type for "arches" parameter: %s' % type(arches)) - fields, aliases = zip(*fields) + fields, aliases = list(zip(*fields)) query = QueryProcessor(columns=fields, aliases=aliases, tables=['rpminfo'], joins=joins, clauses=clauses, values=locals(), transform=_fix_rpm_row, opts=queryOpts) @@ -4118,7 +4120,7 @@ def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hos clauses.append('archiveinfo.btype_id = %(btype_id)s') values['btype_id'] = btype['id'] - columns, aliases = zip(*fields) + columns, aliases = list(zip(*fields)) ret = QueryProcessor(tables=tables, columns=columns, aliases=aliases, joins=joins, transform=_fix_archive_row, clauses=clauses, values=values, opts=queryOpts).execute() @@ -6815,7 +6817,7 @@ def query_history(tables=None, **kwargs): fields[r_test] = '_revoked_before_event' if skip: continue - fields, aliases = zip(*fields.items()) + fields, aliases = list(zip(*fields.items())) query = QueryProcessor(columns=fields, aliases=aliases, tables=[table], joins=joins, clauses=clauses, values=data) ret[table] = query.iterate() @@ -11252,7 +11254,7 @@ class BuildRoot(object): ('checksum_type', 'checksum_type'), ('project_dep', 'project_dep'), ] - columns, aliases = zip(*fields) + columns, aliases = list(zip(*fields)) query = QueryProcessor(tables=tables, columns=columns, joins=joins, clauses=clauses, values=self.data, diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 3eaf46e..3c4a0d4 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -20,6 +20,7 @@ # Mike Bonnet # Mike McLean +from __future__ import absolute_import import os import os.path import re @@ -36,6 +37,7 @@ from kojiweb.util import _initValues from kojiweb.util import _genHTML from kojiweb.util import _getValidTokens from koji.util import sha1_constructor +from six.moves import zip # Convenience definition of a commonly-used sort function _sortbyname = kojiweb.util.sortByKeyFunc('name') From 26661268111ff32a55b0c505769c6a5a9f6e5ff3 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:08:05 +0000 Subject: [PATCH 2/17] python-modernize -f libmodernize.fixes.fix_xrange_six . -w --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index dab88ee..d39c853 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -21,6 +21,7 @@ import six.moves.xmlrpc_client from six.moves import filter from six.moves import map from six.moves import zip +from six.moves import range try: import libcomps diff --git a/hub/kojihub.py b/hub/kojihub.py index 94c1f7f..99c29a9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -60,6 +60,7 @@ import zipfile import koji.xmlrpcplus from koji.context import context from six.moves import zip +from six.moves import range try: import json diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 2307d59..83b7bb4 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -18,6 +18,7 @@ # Authors: # Mike McLean +from __future__ import absolute_import from ConfigParser import RawConfigParser import datetime import inspect @@ -38,6 +39,7 @@ import koji.util # import xmlrpclib functions from koji to use tweaked Marshaller from koji.xmlrpcplus import getparser, dumps, Fault, ExtendedMarshaller from koji.context import context +from six.moves import range class Marshaller(ExtendedMarshaller): diff --git a/koji/context.py b/koji/context.py index 56a79f7..03e22ed 100755 --- a/koji/context.py +++ b/koji/context.py @@ -24,7 +24,9 @@ # - request data # - auth data +from __future__ import absolute_import import thread +from six.moves import range class _data(object): pass @@ -97,7 +99,7 @@ if __name__ == '__main__': context._threadclear() print(context) - for x in xrange(1, 10): + for x in range(1, 10): thread.start_new_thread(test, ()) time.sleep(4) diff --git a/koji/daemon.py b/koji/daemon.py index f279a67..dda3b0a 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -20,6 +20,7 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import import koji import koji.tasks import koji.xmlrpcplus @@ -35,6 +36,7 @@ import time import sys import traceback import errno +from six.moves import range def incremental_upload(session, fname, fd, path, retries=5, logger=None): @@ -694,7 +696,7 @@ class TaskManager(object): fo = open(fn, 'r') id = None name = None - for n in xrange(10): + for n in range(10): # data should be in first few lines line = fo.readline() if line.startswith('# Koji buildroot id:'): diff --git a/koji/tasks.py b/koji/tasks.py index d6521b7..6b40f3d 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -20,6 +20,7 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import import koji import koji.util import os @@ -31,6 +32,7 @@ import random import time import pprint import six.moves.urllib.request +from six.moves import range def scan_mounts(topdir): """Search path for mountpoints""" @@ -424,7 +426,7 @@ class SleepTask(BaseTaskHandler): class ForkTask(BaseTaskHandler): Methods = ['fork'] def handler(self, n=5, m=37): - for i in xrange(n): + for i in range(n): os.spawnvp(os.P_NOWAIT, 'sleep', ['sleep', str(m)]) class WaitTestTask(BaseTaskHandler): @@ -439,7 +441,7 @@ class WaitTestTask(BaseTaskHandler): _taskWeight = 0.1 def handler(self, count, seconds=10): tasks = [] - for i in xrange(count): + for i in range(count): task_id = self.subtask(method='sleep', arglist=[seconds], label=str(i), parent=self.id) tasks.append(task_id) bad_task = self.subtask('sleep', ['BAD_ARG'], label='bad') diff --git a/tests/test_lib/test_plugin.py b/tests/test_lib/test_plugin.py index f064a5d..cfb1168 100644 --- a/tests/test_lib/test_plugin.py +++ b/tests/test_lib/test_plugin.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import datetime import mock @@ -6,6 +7,7 @@ import unittest import koji import koji.util import koji.plugin +from six.moves import range class TestCallbackDecorators(unittest.TestCase): diff --git a/tests/test_lib/test_xmlrpcplus.py b/tests/test_lib/test_xmlrpcplus.py index 70aa7f5..4c5d9a4 100644 --- a/tests/test_lib/test_xmlrpcplus.py +++ b/tests/test_lib/test_xmlrpcplus.py @@ -1,8 +1,10 @@ # coding=utf-8 +from __future__ import absolute_import import unittest from six.moves import xmlrpc_client from koji import xmlrpcplus +from six.moves import range class TestDump(unittest.TestCase): diff --git a/tests/test_lib_py2only/test_tasks.py b/tests/test_lib_py2only/test_tasks.py index 7b2d0f2..f0e22e7 100644 --- a/tests/test_lib_py2only/test_tasks.py +++ b/tests/test_lib_py2only/test_tasks.py @@ -11,6 +11,7 @@ from koji.tasks import BaseTaskHandler, FakeTask, ForkTask, SleepTask, \ WaitTestTask, scan_mounts, umount_all, \ safe_rmtree import six +from six.moves import range def get_fake_mounts_file(): diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 3c4a0d4..1df3070 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -38,6 +38,7 @@ from kojiweb.util import _genHTML from kojiweb.util import _getValidTokens from koji.util import sha1_constructor from six.moves import zip +from six.moves import range # Convenience definition of a commonly-used sort function _sortbyname = kojiweb.util.sortByKeyFunc('name') @@ -812,7 +813,7 @@ def tags(environ, start=None, order=None, childID=None): return _genHTML(environ, 'tags.chtml') -_PREFIX_CHARS = [chr(char) for char in range(48, 58) + range(97, 123)] +_PREFIX_CHARS = [chr(char) for char in list(range(48, 58)) + list(range(97, 123))] def packages(environ, tagID=None, userID=None, order='package_name', start=None, prefix=None, inherited='1'): values = _initValues(environ, 'Packages', 'packages') diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index adea48b..cab820b 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -20,6 +20,7 @@ # Mike Bonnet # Mike McLean +from __future__ import absolute_import import Cheetah.Template import datetime import koji @@ -32,6 +33,7 @@ from socket import sslerror as socket_sslerror from xmlrpclib import ProtocolError from xml.parsers.expat import ExpatError import cgi +from six.moves import range class NoSuchException(Exception): pass From 64f1a8ae36ab7ffb491f3889244717ffb9338eb2 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:09:09 +0000 Subject: [PATCH 3/17] python-modernize -f libmodernize.fixes.fix_imports_six . -w --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 99c29a9..f8974ce 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -54,7 +54,7 @@ import tempfile import traceback import time import types -import xmlrpclib +import six.moves.xmlrpc_client import zipfile import koji.xmlrpcplus @@ -402,7 +402,7 @@ class Task(object): if xml_request.find(' from __future__ import absolute_import -from ConfigParser import RawConfigParser +from six.moves.configparser import RawConfigParser import datetime import inspect import logging diff --git a/koji/compatrequests.py b/koji/compatrequests.py index 2bd89ac..7f12fda 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -7,7 +7,7 @@ module that is based on the older codepaths in koji. It only provides the bits that koji needs. """ -import httplib +import six.moves.http_client import urlparse import urllib import sys @@ -86,11 +86,11 @@ class Session(object): # no verify ctx = pyssl._create_unverified_context() cnxOpts['context'] = ctx - cnxClass = httplib.HTTPSConnection + cnxClass = six.moves.http_client.HTTPSConnection default_port = 443 elif scheme == 'http': cnxOpts = {} - cnxClass = httplib.HTTPConnection + cnxClass = six.moves.http_client.HTTPConnection else: raise IOError("unsupported protocol: %s" % scheme) @@ -125,7 +125,7 @@ class Response(object): def raise_for_status(self): if self.response.status >= 400: - raise httplib.HTTPException("HTTP %s: %s" % (self.response.status, + raise six.moves.http_client.HTTPException("HTTP %s: %s" % (self.response.status, self.response.reason)) diff --git a/koji/context.py b/koji/context.py index 03e22ed..ef35a21 100755 --- a/koji/context.py +++ b/koji/context.py @@ -25,7 +25,7 @@ # - auth data from __future__ import absolute_import -import thread +import six.moves._thread from six.moves import range class _data(object): @@ -37,7 +37,7 @@ class ThreadLocal(object): # should probably be getattribute, but easier to debug this way def __getattr__(self, key): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: raise AttributeError(key) @@ -45,7 +45,7 @@ class ThreadLocal(object): return object.__getattribute__(data, key) def __setattr__(self, key, value): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: tdict[id] = _data() @@ -53,7 +53,7 @@ class ThreadLocal(object): return object.__setattr__(data, key, value) def __delattr__(self, key): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: raise AttributeError(key) @@ -64,14 +64,14 @@ class ThreadLocal(object): return ret def __str__(self): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') return "(current thread: %s) {" % id + \ ", ".join(["%s : %s" %(k, v.__dict__) for (k, v) in tdict.iteritems()]) + \ "}" def _threadclear(self): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: return @@ -100,7 +100,7 @@ if __name__ == '__main__': print(context) for x in range(1, 10): - thread.start_new_thread(test, ()) + six.moves._thread.start_new_thread(test, ()) time.sleep(4) print('') diff --git a/koji/ssl/SSLCommon.py b/koji/ssl/SSLCommon.py index a1ab2bb..cbecedf 100644 --- a/koji/ssl/SSLCommon.py +++ b/koji/ssl/SSLCommon.py @@ -17,7 +17,7 @@ import os, sys from OpenSSL import SSL import SSLConnection -import httplib +import six.moves.http_client import socket def our_verify(connection, x509, errNum, errDepth, preverifyOK): @@ -46,13 +46,13 @@ def CreateSSLContext(certs): return ctx -class PlgHTTPSConnection(httplib.HTTPConnection): +class PlgHTTPSConnection(six.moves.http_client.HTTPConnection): "This class allows communication via SSL." - response_class = httplib.HTTPResponse + response_class = six.moves.http_client.HTTPResponse def __init__(self, host, port=None, ssl_context=None, strict=None, timeout=None): - httplib.HTTPConnection.__init__(self, host, port, strict) + six.moves.http_client.HTTPConnection.__init__(self, host, port, strict) self.ssl_ctx = ssl_context self._timeout = timeout diff --git a/koji/tasks.py b/koji/tasks.py index 6b40f3d..10a74d9 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -25,7 +25,7 @@ import koji import koji.util import os import logging -import xmlrpclib +import six.moves.xmlrpc_client import signal import shutil import random @@ -235,7 +235,7 @@ class BaseTaskHandler(object): continue try: self.session.getTaskResult(task) - except (koji.GenericError, xmlrpclib.Fault) as task_error: + except (koji.GenericError, six.moves.xmlrpc_client.Fault) as task_error: self.logger.info("task %s failed or was canceled" % task) failed = True break diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index 373e880..d128b95 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -2,7 +2,7 @@ import commands import koji -import ConfigParser +import six.moves.configparser import os import platform import re @@ -48,7 +48,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): return res def _read_config(self): - cp = ConfigParser.SafeConfigParser() + cp = six.moves.configparser.SafeConfigParser() cp.read(CONFIG_FILE) self.config = { 'default_mounts': [], @@ -83,7 +83,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): 'fstype': cp.get(section_name, 'fstype'), 'options': cp.get(section_name, 'options'), }) - except ConfigParser.NoOptionError: + except six.moves.configparser.NoOptionError: raise koji.GenericError("bad config: missing options in %s section" % section_name) for path in self.config['default_mounts'] + self.config['safe_roots'] + [x[0] for x in self.config['path_subs']]: diff --git a/plugins/builder/save_failed_tree.py b/plugins/builder/save_failed_tree.py index 4c202c9..b0f5cfd 100644 --- a/plugins/builder/save_failed_tree.py +++ b/plugins/builder/save_failed_tree.py @@ -2,7 +2,7 @@ import fnmatch import os import sys import tarfile -import ConfigParser +import six.moves.configparser import koji import koji.tasks as tasks @@ -27,7 +27,7 @@ def omit_paths3(tarinfo): def read_config(): global config - cp = ConfigParser.SafeConfigParser() + cp = six.moves.configparser.SafeConfigParser() cp.read(CONFIG_FILE) config = { 'path_filters': [], diff --git a/plugins/hub/messagebus.py b/plugins/hub/messagebus.py index 1e208c4..86ac64a 100644 --- a/plugins/hub/messagebus.py +++ b/plugins/hub/messagebus.py @@ -7,7 +7,7 @@ from koji import PluginError from koji.context import context from koji.plugin import callbacks, callback, ignore_error, convert_datetime -import ConfigParser +import six.moves.configparser import logging import qpid.messaging import qpid.messaging.transports @@ -75,7 +75,7 @@ def get_config(): if config: return config - config = ConfigParser.SafeConfigParser() + config = six.moves.configparser.SafeConfigParser() config.read(CONFIG_FILE) if not config.has_option('broker', 'timeout'): config.set('broker', 'timeout', '60') diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index 408e368..d25815a 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -8,7 +8,7 @@ import koji from koji.plugin import callback, ignore_error, convert_datetime from koji.context import context -import ConfigParser +import six.moves.configparser import logging import json import random @@ -259,7 +259,7 @@ def send_queued_msgs(cbtype, *args, **kws): log = logging.getLogger('koji.plugin.protonmsg') global CONFIG if not CONFIG: - conf = ConfigParser.SafeConfigParser() + conf = six.moves.configparser.SafeConfigParser() with open(CONFIG_FILE) as conffile: conf.readfp(conffile) CONFIG = conf diff --git a/plugins/hub/rpm2maven.py b/plugins/hub/rpm2maven.py index ad78b8f..e694d16 100644 --- a/plugins/hub/rpm2maven.py +++ b/plugins/hub/rpm2maven.py @@ -9,7 +9,7 @@ import koji from koji.context import context from koji.plugin import callback from koji.util import rmtree -import ConfigParser +import six.moves.configparser import fnmatch import os import shutil @@ -31,7 +31,7 @@ def maven_import(cbtype, *args, **kws): filepath = kws['filepath'] if not config: - config = ConfigParser.SafeConfigParser() + config = six.moves.configparser.SafeConfigParser() config.read(CONFIG_FILE) name_patterns = config.get('patterns', 'rpm_names').split() for pattern in name_patterns: diff --git a/plugins/hub/save_failed_tree.py b/plugins/hub/save_failed_tree.py index 3a98128..70caa26 100644 --- a/plugins/hub/save_failed_tree.py +++ b/plugins/hub/save_failed_tree.py @@ -1,5 +1,5 @@ import sys -import ConfigParser +import six.moves.configparser import koji from koji.context import context from koji.plugin import export @@ -28,7 +28,7 @@ def saveFailedTree(buildrootID, full=False, **opts): # read configuration only once if config is None: - config = ConfigParser.SafeConfigParser() + config = six.moves.configparser.SafeConfigParser() config.read(CONFIG_FILE) allowed_methods = config.get('permissions', 'allowed_methods').split() if len(allowed_methods) == 1 and allowed_methods[0] == '*': diff --git a/tests/test_lib_py2only/test_compatrequests.py b/tests/test_lib_py2only/test_compatrequests.py index c67f585..6a684ef 100644 --- a/tests/test_lib_py2only/test_compatrequests.py +++ b/tests/test_lib_py2only/test_compatrequests.py @@ -1,4 +1,4 @@ -import httplib +import six.moves.http_client import mock import unittest import urlparse @@ -59,7 +59,7 @@ class TestResponse(unittest.TestCase): self.response.response.status = 404 self.response.response.reason = 'Not Found' self.response.response.getheader.return_value = 42 - with self.assertRaises(httplib.HTTPException): + with self.assertRaises(six.moves.http_client.HTTPException): self.response.raise_for_status() diff --git a/tests/test_plugins/test_protonmsg.py b/tests/test_plugins/test_protonmsg.py index 5b71147..61b1ec5 100644 --- a/tests/test_plugins/test_protonmsg.py +++ b/tests/test_plugins/test_protonmsg.py @@ -4,7 +4,7 @@ import protonmsg from koji.context import context import tempfile from StringIO import StringIO -from ConfigParser import SafeConfigParser +from six.moves.configparser import SafeConfigParser class TestProtonMsg(unittest.TestCase): def tearDown(self): diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 1d40e68..36491fa 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -27,13 +27,13 @@ # in a cygwin shell. from optparse import OptionParser -from ConfigParser import ConfigParser +from six.moves.configparser import ConfigParser import os import subprocess import sys import tempfile import time -import xmlrpclib +import six.moves.xmlrpc_client import base64 import hashlib import logging @@ -586,13 +586,13 @@ def get_mgmt_server(): macaddr, gateway = find_net_info() logger.debug('found MAC address %s, connecting to %s:%s', macaddr, gateway, MANAGER_PORT) - server = xmlrpclib.ServerProxy('http://%s:%s/' % + server = six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, MANAGER_PORT), allow_none=True) # we would set a timeout on the socket here, but that is apparently not # supported by python/cygwin/Windows task_port = server.getPort(macaddr) logger.debug('found task-specific port %s', task_port) - return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + return six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) def get_options(): """handle usage and parse options""" diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 1df3070..aa5708d 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -26,7 +26,7 @@ import os.path import re import sys import mimetypes -import Cookie +import six.moves.http_cookies import datetime import logging import time @@ -56,7 +56,7 @@ def _setUserCookie(environ, user): shasum = sha1_constructor(value) shasum.update(options['Secret'].value) value = "%s:%s" % (shasum.hexdigest(), value) - cookies = Cookie.SimpleCookie() + cookies = six.moves.http_cookies.SimpleCookie() cookies['user'] = value c = cookies['user'] #morsel instance c['secure'] = True @@ -69,7 +69,7 @@ def _setUserCookie(environ, user): environ['koji.headers'].append(['Cache-Control', 'no-cache="set-cookie"']) def _clearUserCookie(environ): - cookies = Cookie.SimpleCookie() + cookies = six.moves.http_cookies.SimpleCookie() cookies['user'] = '' c = cookies['user'] #morsel instance c['path'] = os.path.dirname(environ['SCRIPT_NAME']) @@ -79,7 +79,7 @@ def _clearUserCookie(environ): def _getUserCookie(environ): options = environ['koji.options'] - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) + cookies = six.moves.http_cookies.SimpleCookie(environ.get('HTTP_COOKIE', '')) if 'user' not in cookies: return None value = cookies['user'].value diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 778d992..98d75f6 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -29,7 +29,7 @@ import pprint import sys import traceback -from ConfigParser import RawConfigParser +from six.moves.configparser import RawConfigParser from koji.server import ServerError, ServerRedirect from koji.util import dslice diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index cab820b..e6b991f 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -30,7 +30,7 @@ import stat #a bunch of exception classes that explainError needs from socket import error as socket_error from socket import sslerror as socket_sslerror -from xmlrpclib import ProtocolError +from six.moves.xmlrpc_client import ProtocolError from xml.parsers.expat import ExpatError import cgi from six.moves import range From 9a0a63739307520ff664f602b2a5fe79312e1ccd Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:11:29 +0000 Subject: [PATCH 4/17] python-modernize -f libmodernize.fixes.fix_dict_six . -w --- diff --git a/hub/kojihub.py b/hub/kojihub.py index f8974ce..b3a023f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -61,6 +61,7 @@ import koji.xmlrpcplus from koji.context import context from six.moves import zip from six.moves import range +import six try: import json @@ -671,12 +672,12 @@ def _writeInheritanceData(tag_id, changes, clear=False): data[parent_id] = link break if clear: - for link in data.itervalues(): + for link in six.itervalues(data): if not link.get('is_update'): link['delete link'] = True link['is_update'] = True changed = False - for link in data.itervalues(): + for link in six.itervalues(data): if link.get('is_update'): changed = True break @@ -686,17 +687,17 @@ def _writeInheritanceData(tag_id, changes, clear=False): return #check for duplicate priorities pri_index = {} - for link in data.itervalues(): + for link in six.itervalues(data): if link.get('delete link'): continue pri_index.setdefault(link['priority'], []).append(link) - for pri, dups in pri_index.iteritems(): + for pri, dups in six.iteritems(pri_index): if len(dups) <= 1: continue #oops, duplicate entries for a single priority dup_ids = [link['parent_id'] for link in dups] raise koji.GenericError("Inheritance priorities must be unique (pri %s: %r )" % (pri, dup_ids)) - for parent_id, link in data.iteritems(): + for parent_id, link in six.iteritems(data): if not link.get('is_update'): continue # revoke old values @@ -704,7 +705,7 @@ def _writeInheritanceData(tag_id, changes, clear=False): clauses=['tag_id=%(tag_id)s', 'parent_id = %(parent_id)s']) update.make_revoke() update.execute() - for parent_id, link in data.iteritems(): + for parent_id, link in six.iteritems(data): if not link.get('is_update'): continue # skip rest if we are just deleting @@ -1973,7 +1974,7 @@ def get_tag_groups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True groups.setdefault(grp_id, group) if incl_pkgs: - for group in groups.itervalues(): + for group in six.itervalues(groups): group['packagelist'] = {} fields = ('group_id', 'tag_id', 'package', 'blocked', 'type', 'basearchonly', 'requires') q = """ @@ -1995,7 +1996,7 @@ def get_tag_groups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True if incl_reqs: # and now the group reqs - for group in groups.itervalues(): + for group in six.itervalues(groups): group['grouplist'] = {} fields = ('group_id', 'tag_id', 'req_id', 'blocked', 'type', 'is_metapkg', 'name') q = """SELECT %s FROM group_req_listing JOIN groups on req_id = id @@ -2161,7 +2162,7 @@ def get_all_arches(): #in a perfect world, this list would only include canonical #arches, but not all admins will undertand that. ret[koji.canonArch(arch)] = 1 - return ret.keys() + return list(ret.keys()) def get_active_tasks(host=None): """Return data on tasks that are yet to be run""" @@ -2414,7 +2415,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): os.symlink(relpath, destlink) except: log_error('Error linking %s to %s' % (destlink, relpath)) - for artifact_dir, artifacts in artifact_dirs.iteritems(): + for artifact_dir, artifacts in six.iteritems(artifact_dirs): _write_maven_repo_metadata(artifact_dir, artifacts) koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo, @@ -2553,7 +2554,7 @@ def repo_references(repo_id): 'host_id': 'host_id', 'create_event': 'create_event', 'state': 'state'} - fields, aliases = list(zip(*fields.items())) + fields, aliases = list(zip(*list(fields.items()))) values = {'repo_id': repo_id} clauses = ['repo_id=%(repo_id)s', 'retire_event IS NULL'] query = QueryProcessor(columns=fields, aliases=aliases, tables=['standard_buildroot'], @@ -2922,7 +2923,7 @@ def _create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_s # add extra data if extra is not None: - for key, value in extra.iteritems(): + for key, value in six.iteritems(extra): data = { 'tag_id': tag_id, 'key': key, @@ -2987,7 +2988,7 @@ def get_tag(tagInfo, strict=False, event=None): raise koji.GenericError('invalid type for tagInfo: %s' % type(tagInfo)) data = {'tagInfo': tagInfo} - fields, aliases = list(zip(*fields.items())) + fields, aliases = list(zip(*list(fields.items()))) query = QueryProcessor(columns=fields, aliases=aliases, tables=tables, joins=joins, clauses=clauses, values=data) result = query.executeOne() @@ -5048,7 +5049,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) policy_data = { 'package': build['name'], - 'buildroots': brmap.values(), + 'buildroots': list(brmap.values()), 'import': True, 'import_type': 'rpm', } @@ -5088,7 +5089,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) import_rpm_file(fn, binfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) if logs: - for key, files in logs.iteritems(): + for key, files in six.iteritems(logs): if not key: key = None for relpath in files: @@ -6818,7 +6819,7 @@ def query_history(tables=None, **kwargs): fields[r_test] = '_revoked_before_event' if skip: continue - fields, aliases = list(zip(*fields.items())) + fields, aliases = list(zip(*list(fields.items()))) query = QueryProcessor(columns=fields, aliases=aliases, tables=[table], joins=joins, clauses=clauses, values=data) ret[table] = query.iterate() @@ -6957,7 +6958,7 @@ def build_references(build_id, limit=None): idx.setdefault(row['id'], row) if limit is not None and len(idx) > limit: break - ret['rpms'] = idx.values() + ret['rpms'] = list(idx.values()) ret['component_of'] = [] # find images/archives that contain the build rpms @@ -6988,7 +6989,7 @@ def build_references(build_id, limit=None): idx.setdefault(row['id'], row) if limit is not None and len(idx) > limit: break - ret['archives'] = idx.values() + ret['archives'] = list(idx.values()) # find images/archives that contain the build archives fields = ['archive_id'] @@ -7294,7 +7295,7 @@ def get_notification_recipients(build, tag_id, state): #FIXME - if tag_id is None, we don't have a good way to get the package owner. # using all package owners from all tags would be way overkill. - emails_uniq = dict([(x, 1) for x in emails]).keys() + emails_uniq = list(dict([(x, 1) for x in emails]).keys()) return emails_uniq def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_success=False, failure_msg=''): @@ -7317,7 +7318,7 @@ def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_s from_tag = get_tag(from_id) for email in get_notification_recipients(build, from_tag['id'], state): recipients[email] = 1 - recipients_uniq = recipients.keys() + recipients_uniq = list(recipients.keys()) if len(recipients_uniq) > 0 and not (is_successful and ignore_success): task_id = make_task('tagNotification', [recipients_uniq, is_successful, tag_id, from_id, build_id, user_id, ignore_success, failure_msg]) return task_id @@ -7535,8 +7536,8 @@ class InsertProcessor(object): if not self.data and not self.rawdata: return "-- incomplete update: no assigns" parts = ['INSERT INTO %s ' % self.table] - columns = self.data.keys() - columns.extend(self.rawdata.keys()) + columns = list(self.data.keys()) + columns.extend(list(self.rawdata.keys())) parts.append("(%s) " % ', '.join(columns)) values = [] for key in columns: @@ -7579,7 +7580,7 @@ class InsertProcessor(object): del data['create_event'] del data['creator_id'] clauses = ["%s = %%(%s)s" % (k, k) for k in data] - query = QueryProcessor(columns=data.keys(), tables=[self.table], + query = QueryProcessor(columns=list(data.keys()), tables=[self.table], clauses=clauses, values=data) if query.execute(): return True @@ -8246,7 +8247,7 @@ class UserInGroupTest(koji.policy.BaseSimpleTest): return False groups = koji.auth.get_user_groups(user['id']) args = self.str.split()[1:] - for group_id, group in groups.iteritems(): + for group_id, group in six.iteritems(groups): for pattern in args: if fnmatch.fnmatch(group, pattern): return True @@ -10061,9 +10062,9 @@ class RootExports(object): userID = get_user(userID, strict=True)['id'] if pkgID is not None: pkgID = get_package_id(pkgID, strict=True) - result_list = readPackageList(tagID=tagID, userID=userID, pkgID=pkgID, + result_list = list(readPackageList(tagID=tagID, userID=userID, pkgID=pkgID, inherit=inherited, with_dups=with_dups, - event=event).values() + event=event).values()) if with_dups: # when with_dups=True, readPackageList returns a list of list of dicts # convert it to a list of dicts for consistency @@ -11667,7 +11668,7 @@ class HostExports(object): safer_move(fn, dest) os.symlink(dest, fn) if logs: - for key, files in logs.iteritems(): + for key, files in six.iteritems(logs): if key: logdir = "%s/logs/%s" % (dir, key) else: @@ -11690,7 +11691,7 @@ class HostExports(object): scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] destdir = os.path.join(scratchdir, username, 'task_%s' % task_id) - for reldir, files in results['files'].items() + [('', results['logs'])]: + for reldir, files in list(results['files'].items()) + [('', results['logs'])]: for filename in files: if reldir: relpath = os.path.join(reldir, filename) @@ -11722,7 +11723,7 @@ class HostExports(object): scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] destdir = os.path.join(scratchdir, username, 'task_%s' % task_id) - for relpath in results['output'].keys() + results['logs']: + for relpath in list(results['output'].keys()) + results['logs']: filename = os.path.join(koji.pathinfo.task(results['task_id']), relpath) dest = os.path.join(destdir, relpath) koji.ensuredir(os.path.dirname(dest)) @@ -11911,7 +11912,7 @@ class HostExports(object): maven_task_id = maven_results['task_id'] maven_buildroot_id = maven_results['buildroot_id'] maven_task_dir = koji.pathinfo.task(maven_task_id) - for relpath, files in maven_results['files'].iteritems(): + for relpath, files in six.iteritems(maven_results['files']): dir_maven_info = maven_info poms = [f for f in files if f.endswith('.pom')] if len(poms) == 0: @@ -12075,7 +12076,7 @@ class HostExports(object): task_dir = koji.pathinfo.task(results['task_id']) # import the build output - for relpath, metadata in results['output'].iteritems(): + for relpath, metadata in six.iteritems(results['output']): archivetype = get_archive_type(relpath) if not archivetype: # Unknown archive type, fail the build @@ -12303,7 +12304,7 @@ class HostExports(object): for dep in extra_deps: if isinstance(dep, (int, long)): task_output = list_task_output(dep, stat=True) - for filepath, filestats in task_output.iteritems(): + for filepath, filestats in six.iteritems(task_output): if os.path.splitext(filepath)[1] in ['.log', '.md5', '.sha1']: continue tokens = filepath.split('/') @@ -12336,7 +12337,7 @@ class HostExports(object): logger.error("Current build is %s, new build is %s.", idx_build, archive['build_id']) maven_build_index[archive['group_id']][archive['artifact_id']][archive['version']] = archive['build_id'] - ignore.extend(task_deps.values()) + ignore.extend(list(task_deps.values())) SNAPSHOT_RE = re.compile(r'-\d{8}\.\d{6}-\d+') ignore_by_label = {} @@ -12389,7 +12390,7 @@ class HostExports(object): if build_id: build = get_build(build_id) logger.error("g:a:v supplied by build %(nvr)s", build) - logger.error("Build supplies %i archives: %r", len(build_archives), build_archives.keys()) + logger.error("Build supplies %i archives: %r", len(build_archives), list(build_archives.keys())) if tag_archive: logger.error("Size mismatch, br: %i, db: %i", fileinfo['size'], tag_archive['size']) raise koji.BuildrootError('Unknown file in build environment: %s, size: %s' % \ @@ -12467,7 +12468,7 @@ class HostExports(object): repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() if not rinfo['dist']: - for arch, (uploadpath, files) in data.iteritems(): + for arch, (uploadpath, files) in six.iteritems(data): archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): raise koji.GenericError("Repo arch directory missing: %s" % archdir) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 88925a1..5aeb583 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -40,6 +40,7 @@ import koji.util from koji.xmlrpcplus import getparser, dumps, Fault, ExtendedMarshaller from koji.context import context from six.moves import range +import six class Marshaller(ExtendedMarshaller): @@ -98,7 +99,7 @@ class HandlerRegistry(object): Handlers are functions marked with one of the decorators defined in koji.plugin """ - for v in vars(plugin).itervalues(): + for v in six.itervalues(vars(plugin)): if isinstance(v, type): #skip classes continue @@ -155,7 +156,7 @@ class HandlerRegistry(object): return args def system_listMethods(self): - return self.funcs.keys() + return list(self.funcs.keys()) def system_methodSignature(self, method): #it is not possible to autogenerate this data @@ -477,7 +478,7 @@ def load_config(environ): opts['policy'] = dict(config.items('policy')) else: opts['policy'] = {} - for pname, text in _default_policies.iteritems(): + for pname, text in six.iteritems(_default_policies): opts['policy'].setdefault(pname, text) # use configured KojiDir if opts.get('KojiDir') is not None: @@ -545,12 +546,12 @@ def get_policy(opts, plugins): continue alltests.append(koji.policy.findSimpleTests(vars(plugin))) policy = {} - for pname, text in opts['policy'].iteritems(): + for pname, text in six.iteritems(opts['policy']): #filter/merge tests merged = {} for tests in alltests: # tests can be limited to certain policies by setting a class variable - for name, test in tests.iteritems(): + for name, test in six.iteritems(tests): if hasattr(test, 'policy'): if isinstance(test.policy, basestring): if pname != test.policy: diff --git a/koji/auth.py b/koji/auth.py index 6f43159..706f6bc 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -104,7 +104,7 @@ class Session(object): 'user_id': 'user_id', } # sort for stability (unittests) - fields, aliases = list(zip(*list(sorted(fields.items(), key=lambda x: x[1])))) + fields, aliases = list(zip(*list(sorted(list(fields.items()), key=lambda x: x[1])))) q = """ SELECT %s FROM sessions WHERE id = %%(id)i diff --git a/koji/context.py b/koji/context.py index ef35a21..707c626 100755 --- a/koji/context.py +++ b/koji/context.py @@ -27,6 +27,7 @@ from __future__ import absolute_import import six.moves._thread from six.moves import range +import six class _data(object): pass @@ -67,7 +68,7 @@ class ThreadLocal(object): id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') return "(current thread: %s) {" % id + \ - ", ".join(["%s : %s" %(k, v.__dict__) for (k, v) in tdict.iteritems()]) + \ + ", ".join(["%s : %s" %(k, v.__dict__) for (k, v) in six.iteritems(tdict)]) + \ "}" def _threadclear(self): diff --git a/koji/daemon.py b/koji/daemon.py index dda3b0a..c41d2f2 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -37,6 +37,7 @@ import sys import traceback import errno from six.moves import range +import six def incremental_upload(session, fname, fd, path, retries=5, logger=None): @@ -557,7 +558,7 @@ class TaskManager(object): """Attempt to shut down cleanly""" for task_id in self.pids.keys(): self.cleanupTask(task_id) - self.session.host.freeTasks(self.tasks.keys()) + self.session.host.freeTasks(list(self.tasks.keys())) self.session.host.updateHost(task_load=0.0, ready=False) def updateBuildroots(self, nolocal=False): @@ -588,14 +589,14 @@ class TaskManager(object): #task not running - expire the buildroot #TODO - consider recycling hooks here (with strong sanity checks) self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br) - self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, self.tasks.keys())) + self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, list(self.tasks.keys()))) self.session.host.setBuildRootState(id, st_expired) continue if nolocal: return local_br = self._scanLocalBuildroots() # get info on local_only buildroots (most likely expired) - local_only = [id for id in local_br.iterkeys() if id not in db_br] + local_only = [id for id in six.iterkeys(local_br) if id not in db_br] if local_only: missed_br = self.session.listBuildroots(buildrootID=tuple(local_only)) #get all the task info in one call @@ -827,7 +828,7 @@ class TaskManager(object): # Note: we may still take an assigned task below #sort available capacities for each of our bins avail = {} - for bin in bins.iterkeys(): + for bin in six.iterkeys(bins): avail[bin] = [host['capacity'] - host['task_load'] for host in bin_hosts[bin]] avail[bin].sort() avail[bin].reverse() diff --git a/koji/policy.py b/koji/policy.py index 0842d84..5cb2047 100644 --- a/koji/policy.py +++ b/koji/policy.py @@ -17,10 +17,12 @@ # Authors: # Mike McLean +from __future__ import absolute_import import fnmatch import logging import koji +import six class BaseSimpleTest(object): @@ -292,7 +294,7 @@ class SimpleRuleSet(object): index[name] = 1 index = {} _recurse(self.ruleset, index) - return index.keys() + return list(index.keys()) def _apply(self, rules, data, top=False): for tests, negate, action in rules: @@ -361,7 +363,7 @@ def findSimpleTests(namespace): namespace = (namespace,) ret = {} for ns in namespace: - for key, value in ns.iteritems(): + for key, value in six.iteritems(ns): if value is BaseSimpleTest: # skip this abstract base class if we encounter it # this module contains generic tests, so it is valid to include it diff --git a/plugins/hub/echo.py b/plugins/hub/echo.py index 6727d41..38c96ba 100644 --- a/plugins/hub/echo.py +++ b/plugins/hub/echo.py @@ -8,7 +8,7 @@ from koji.plugin import callbacks, callback, ignore_error import logging -@callback(*callbacks.keys()) +@callback(*list(callbacks.keys())) @ignore_error def echo(cbtype, *args, **kws): logging.getLogger('koji.plugin.echo').info('Called the %s callback, args: %s; kws: %s', diff --git a/tests/test_lib/test_utils.py b/tests/test_lib/test_utils.py index fa94afe..18b5225 100644 --- a/tests/test_lib/test_utils.py +++ b/tests/test_lib/test_utils.py @@ -12,6 +12,7 @@ import calendar import six.moves.configparser import koji import koji.util +import six class EnumTestCase(unittest.TestCase): @@ -764,24 +765,24 @@ class MavenUtilTestCase(unittest.TestCase): name, release, date = 'fedora', 26, datetime.now().strftime('%Y%m%d') data = {'name': name, 'release': release, 'date': date} - six.assertCountEqual(self, data.items(), ldict.items()) - six.assertCountEqual(self, data.items(), [v for v in ldict.iteritems()]) + six.assertCountEqual(self, list(data.items()), list(ldict.items())) + six.assertCountEqual(self, list(data.items()), [v for v in six.iteritems(ldict)]) name, release, date = 'rhel', 7, '20171012' - six.assertCountEqual(self, [name, release, date], ldict.values()) - six.assertCountEqual(self, [name, release, date], [v for v in ldict.itervalues()]) + six.assertCountEqual(self, [name, release, date], list(ldict.values())) + six.assertCountEqual(self, [name, release, date], [v for v in six.itervalues(ldict)]) data = {'name': name, 'release': release, 'date': date} self.assertEqual(name, ldict.pop('name')) data.pop('name') - six.assertCountEqual(self, data.items(), ldict.items()) + six.assertCountEqual(self, list(data.items()), list(ldict.items())) (key, value) = ldict.popitem() data.pop(key) - six.assertCountEqual(self, data.items(), ldict.items()) + six.assertCountEqual(self, list(data.items()), list(ldict.items())) ldict_copy = ldict.copy() - six.assertCountEqual(self, data.items(), ldict_copy.items()) + six.assertCountEqual(self, list(data.items()), list(ldict_copy.items())) def test_LazyRecord(self): """Test LazyRecord object""" @@ -858,7 +859,7 @@ class MavenUtilTestCase(unittest.TestCase): actual = koji.util.eventFromOpts(session, opts) self.assertNotEqual(None, actual) - six.assertCountEqual(self, expect.items(), actual.items()) + six.assertCountEqual(self, list(expect.items()), list(actual.items())) # no event is matched case opts = mock.MagicMock(event=0, ts=0, repo=0) diff --git a/tests/test_plugins/test_runroot_builder.py b/tests/test_plugins/test_runroot_builder.py index 21ffc12..c300a60 100644 --- a/tests/test_plugins/test_runroot_builder.py +++ b/tests/test_plugins/test_runroot_builder.py @@ -69,7 +69,7 @@ class FakeConfigParser(object): return def sections(self): - return self.CONFIG.keys() + return list(self.CONFIG.keys()) def has_option(self, section, key): return section in self.CONFIG and key in self.CONFIG[section] diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 36491fa..fe70dd6 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -26,6 +26,7 @@ # kojiwind --install # in a cygwin shell. +from __future__ import absolute_import from optparse import OptionParser from six.moves.configparser import ConfigParser import os @@ -42,6 +43,7 @@ import threading import re import glob import zipfile +import six MANAGER_PORT = 7000 @@ -639,7 +641,7 @@ def stream_logs(server, handler, builds): logpath = os.path.join(build.source_dir, relpath) if logpath not in logs: logs[logpath] = (relpath, None) - for log, (relpath, fd) in logs.iteritems(): + for log, (relpath, fd) in six.iteritems(logs): if not fd: if os.path.isfile(log): try: diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index aa5708d..c003d51 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -39,6 +39,7 @@ from kojiweb.util import _getValidTokens from koji.util import sha1_constructor from six.moves import zip from six.moves import range +import six # Convenience definition of a commonly-used sort function _sortbyname = kojiweb.util.sortByKeyFunc('name') @@ -684,7 +685,7 @@ def taskinfo(environ, taskID): values['pathinfo'] = pathinfo paths = [] # (volume, relpath) tuples - for relname, volumes in server.listTaskOutput(task['id'], all_volumes=True).iteritems(): + for relname, volumes in six.iteritems(server.listTaskOutput(task['id'], all_volumes=True)): paths += [(volume, relname) for volume in volumes] values['output'] = sorted(paths, cmp = _sortByExtAndName) if environ['koji.currentUser']: @@ -703,8 +704,8 @@ def taskstatus(environ, taskID): return '' files = server.listTaskOutput(taskID, stat=True, all_volumes=True) output = '%i:%s\n' % (task['id'], koji.TASK_STATES[task['state']]) - for filename, volumes_data in files.iteritems(): - for volume, file_stats in volumes_data.iteritems(): + for filename, volumes_data in six.iteritems(files): + for volume, file_stats in six.iteritems(volumes_data): output += '%s:%s:%s\n' % (volume, filename, file_stats['st_size']) return output @@ -2108,7 +2109,7 @@ def buildsbytarget(environ, days='7', start=None, order='-builds'): if builds > maxBuilds: maxBuilds = builds - kojiweb.util.paginateList(values, targets.values(), start, 'targets', 'target', order) + kojiweb.util.paginateList(values, list(targets.values()), start, 'targets', 'target', order) values['order'] = order diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 98d75f6..a2c9a6d 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -399,7 +399,7 @@ class Dispatcher(object): if isinstance(result, basestring): headers.setdefault('content-length', ('Content-Length', str(len(result)))) headers.setdefault('content-type', ('Content-Type', 'text/html')) - headers = headers.values() + extra + headers = list(headers.values()) + extra self.logger.debug("Headers:") self.logger.debug(koji.util.LazyString(pprint.pformat, [headers])) start_response(status, headers) From f4010ea0747c90c1528765df3aa023f91efe9911 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:15:18 +0000 Subject: [PATCH 5/17] python-modernize -f libmodernize.fixes.fix_import . -w --- diff --git a/koji/compatrequests.py b/koji/compatrequests.py index 7f12fda..a3e9aa2 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -7,14 +7,15 @@ module that is based on the older codepaths in koji. It only provides the bits that koji needs. """ +from __future__ import absolute_import import six.moves.http_client import urlparse import urllib import sys -import ssl.SSLCommon +from . import ssl.SSLCommon from warnings import warn try: - from ssl import ssl as pyssl + from .ssl import ssl as pyssl except ImportError: # pragma: no cover pass diff --git a/koji/db.py b/koji/db.py index 9e92191..c4e9ab3 100644 --- a/koji/db.py +++ b/koji/db.py @@ -21,6 +21,7 @@ # Mike McLean +from __future__ import absolute_import import logging import sys import psycopg2 @@ -33,7 +34,7 @@ import psycopg2 # del psycopg2.extensions.string_types[1266] import time import traceback -import context +from . import context import re POSITIONAL_RE = re.compile(r'%[a-z]') diff --git a/koji/plugin.py b/koji/plugin.py index 396b811..5250e87 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -19,6 +19,7 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import import imp import koji import logging diff --git a/koji/ssl/SSLCommon.py b/koji/ssl/SSLCommon.py index cbecedf..0cff1ee 100644 --- a/koji/ssl/SSLCommon.py +++ b/koji/ssl/SSLCommon.py @@ -14,9 +14,10 @@ # # Copyright 2005 Dan Williams and Red Hat, Inc. +from __future__ import absolute_import import os, sys from OpenSSL import SSL -import SSLConnection +from . import SSLConnection import six.moves.http_client import socket diff --git a/koji/ssl/SSLConnection.py b/koji/ssl/SSLConnection.py index 2e53db7..92de572 100644 --- a/koji/ssl/SSLConnection.py +++ b/koji/ssl/SSLConnection.py @@ -6,6 +6,7 @@ # Modifications by Dan Williams +from __future__ import absolute_import from OpenSSL import SSL import time, socket, select diff --git a/koji/ssl/__init__.py b/koji/ssl/__init__.py index 7b4c2b3..fb2e3c7 100644 --- a/koji/ssl/__init__.py +++ b/koji/ssl/__init__.py @@ -1,6 +1,7 @@ # identify this as the ssl module # our own ssl submodule masks python's in the main lib, so we import this here +from __future__ import absolute_import try: import ssl # python's ssl module except ImportError: # pragma: no cover diff --git a/koji/xmlrpcplus.py b/koji/xmlrpcplus.py index 9164cd7..06284b4 100644 --- a/koji/xmlrpcplus.py +++ b/koji/xmlrpcplus.py @@ -2,6 +2,7 @@ Custom xmlrpc handling for Koji """ +from __future__ import absolute_import import six import six.moves.xmlrpc_client as xmlrpc_client import types diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index d128b95..1ab738d 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -1,5 +1,6 @@ # kojid plugin +from __future__ import absolute_import import commands import koji import six.moves.configparser diff --git a/plugins/builder/save_failed_tree.py b/plugins/builder/save_failed_tree.py index b0f5cfd..7c579a7 100644 --- a/plugins/builder/save_failed_tree.py +++ b/plugins/builder/save_failed_tree.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import fnmatch import os import sys diff --git a/plugins/cli/runroot.py b/plugins/cli/runroot.py index f8d4b50..a1a05ee 100644 --- a/plugins/cli/runroot.py +++ b/plugins/cli/runroot.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import sys import time diff --git a/plugins/cli/save_failed_tree.py b/plugins/cli/save_failed_tree.py index 46b5fa3..5bf1a6d 100644 --- a/plugins/cli/save_failed_tree.py +++ b/plugins/cli/save_failed_tree.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import koji from koji.plugin import export_cli from koji_cli.lib import _, activate_session, OptionParser, watch_tasks diff --git a/plugins/hub/echo.py b/plugins/hub/echo.py index 38c96ba..739dfeb 100644 --- a/plugins/hub/echo.py +++ b/plugins/hub/echo.py @@ -5,6 +5,7 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import from koji.plugin import callbacks, callback, ignore_error import logging diff --git a/plugins/hub/messagebus.py b/plugins/hub/messagebus.py index 86ac64a..d6601e9 100644 --- a/plugins/hub/messagebus.py +++ b/plugins/hub/messagebus.py @@ -4,6 +4,7 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import from koji import PluginError from koji.context import context from koji.plugin import callbacks, callback, ignore_error, convert_datetime diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index d25815a..57e108b 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -5,6 +5,7 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import import koji from koji.plugin import callback, ignore_error, convert_datetime from koji.context import context diff --git a/plugins/hub/rpm2maven.py b/plugins/hub/rpm2maven.py index e694d16..717df54 100644 --- a/plugins/hub/rpm2maven.py +++ b/plugins/hub/rpm2maven.py @@ -5,6 +5,7 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import import koji from koji.context import context from koji.plugin import callback diff --git a/plugins/hub/runroot_hub.py b/plugins/hub/runroot_hub.py index e6ee0b3..708aede 100644 --- a/plugins/hub/runroot_hub.py +++ b/plugins/hub/runroot_hub.py @@ -3,6 +3,7 @@ # plugin has a config file. This hub plugin has no config file. +from __future__ import absolute_import from koji.context import context from koji.plugin import export import koji diff --git a/plugins/hub/save_failed_tree.py b/plugins/hub/save_failed_tree.py index 70caa26..70a796f 100644 --- a/plugins/hub/save_failed_tree.py +++ b/plugins/hub/save_failed_tree.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import sys import six.moves.configparser import koji diff --git a/tests/test_cli/data/plugins/plugin1.py b/tests/test_cli/data/plugins/plugin1.py index 3e405f5..41d876e 100644 --- a/tests/test_cli/data/plugins/plugin1.py +++ b/tests/test_cli/data/plugins/plugin1.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from koji.plugin import export_cli, export_as @export_as('foobar') diff --git a/tests/test_cli/test_activate_session.py b/tests/test_cli/test_activate_session.py index 987f08d..51ce645 100644 --- a/tests/test_cli/test_activate_session.py +++ b/tests/test_cli/test_activate_session.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import shutil import tempfile diff --git a/tests/test_cli/test_list_tasks.py b/tests/test_cli/test_list_tasks.py index 01d4cb5..550504e 100644 --- a/tests/test_cli/test_list_tasks.py +++ b/tests/test_cli/test_list_tasks.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_docs_version.py b/tests/test_docs_version.py index 21beb85..03711a1 100644 --- a/tests/test_docs_version.py +++ b/tests/test_docs_version.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os import subprocess import unittest diff --git a/tests/test_hub/test_add_btype.py b/tests/test_hub/test_add_btype.py index e64cc0f..6a73b0d 100644 --- a/tests/test_hub/test_add_btype.py +++ b/tests/test_hub/test_add_btype.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_apply_query_opts.py b/tests/test_hub/test_apply_query_opts.py index 653723c..6d23d6a 100644 --- a/tests/test_hub/test_apply_query_opts.py +++ b/tests/test_hub/test_apply_query_opts.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import unittest diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index ac0e528..708a464 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock import os diff --git a/tests/test_hub/test_check_volume_policy.py b/tests/test_hub/test_check_volume_policy.py index e8c9843..64c0f8a 100644 --- a/tests/test_hub/test_check_volume_policy.py +++ b/tests/test_hub/test_check_volume_policy.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_complete_image_build.py b/tests/test_hub/test_complete_image_build.py index 0f67b15..2742afd 100644 --- a/tests/test_hub/test_complete_image_build.py +++ b/tests/test_hub/test_complete_image_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import json import mock diff --git a/tests/test_hub/test_complete_maven_build.py b/tests/test_hub/test_complete_maven_build.py index 45ec01c..bbce516 100644 --- a/tests/test_hub/test_complete_maven_build.py +++ b/tests/test_hub/test_complete_maven_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import json import mock diff --git a/tests/test_hub/test_create_tag.py b/tests/test_hub/test_create_tag.py index 91dea25..4635658 100644 --- a/tests/test_hub/test_create_tag.py +++ b/tests/test_hub/test_create_tag.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock import shutil diff --git a/tests/test_hub/test_delete_build.py b/tests/test_hub/test_delete_build.py index 2cd2dae..91e497f 100644 --- a/tests/test_hub/test_delete_build.py +++ b/tests/test_hub/test_delete_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest import kojihub diff --git a/tests/test_hub/test_delete_tag.py b/tests/test_hub/test_delete_tag.py index 76875e2..45fd944 100644 --- a/tests/test_hub/test_delete_tag.py +++ b/tests/test_hub/test_delete_tag.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py index 33f1b6c..50b6057 100644 --- a/tests/test_hub/test_dist_repo.py +++ b/tests/test_hub/test_dist_repo.py @@ -1,4 +1,5 @@ +from __future__ import absolute_import import unittest import mock import os diff --git a/tests/test_hub/test_edit_tag.py b/tests/test_hub/test_edit_tag.py index 65d6dc6..0c57e75 100644 --- a/tests/test_hub/test_edit_tag.py +++ b/tests/test_hub/test_edit_tag.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_get_active_repos.py b/tests/test_hub/test_get_active_repos.py index da6c325..826a99e 100644 --- a/tests/test_hub/test_get_active_repos.py +++ b/tests/test_hub/test_get_active_repos.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_get_build_logs.py b/tests/test_hub/test_get_build_logs.py index 88157aa..8add61f 100644 --- a/tests/test_hub/test_get_build_logs.py +++ b/tests/test_hub/test_get_build_logs.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import os import shutil diff --git a/tests/test_hub/test_get_build_type.py b/tests/test_hub/test_get_build_type.py index b4013e9..e4a0862 100644 --- a/tests/test_hub/test_get_build_type.py +++ b/tests/test_hub/test_get_build_type.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_get_next_release.py b/tests/test_hub/test_get_next_release.py index adfeea3..92a3e14 100644 --- a/tests/test_hub/test_get_next_release.py +++ b/tests/test_hub/test_get_next_release.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_get_upload_path.py b/tests/test_hub/test_get_upload_path.py index a1191ce..d11e1b1 100644 --- a/tests/test_hub/test_get_upload_path.py +++ b/tests/test_hub/test_get_upload_path.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os import mock import shutil diff --git a/tests/test_hub/test_get_user_perms.py b/tests/test_hub/test_get_user_perms.py index 5112083..0c60149 100644 --- a/tests/test_hub/test_get_user_perms.py +++ b/tests/test_hub/test_get_user_perms.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_get_verify_class.py b/tests/test_hub/test_get_verify_class.py index ae94ed6..57809e8 100644 --- a/tests/test_hub/test_get_verify_class.py +++ b/tests/test_hub/test_get_verify_class.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import kojihub from koji import GenericError diff --git a/tests/test_hub/test_group_operations.py b/tests/test_hub/test_group_operations.py index c62cab4..bde4159 100644 --- a/tests/test_hub/test_group_operations.py +++ b/tests/test_hub/test_group_operations.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index a0d1e56..1c329d9 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock import shutil diff --git a/tests/test_hub/test_import_image_internal.py b/tests/test_hub/test_import_image_internal.py index 75dedf4..e80037f 100644 --- a/tests/test_hub/test_import_image_internal.py +++ b/tests/test_hub/test_import_image_internal.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock import os diff --git a/tests/test_hub/test_insert_processor.py b/tests/test_hub/test_insert_processor.py index 1771681..fde1cc3 100644 --- a/tests/test_hub/test_insert_processor.py +++ b/tests/test_hub/test_insert_processor.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_list_btypes.py b/tests/test_hub/test_list_btypes.py index b5f4bc5..6649e89 100644 --- a/tests/test_hub/test_list_btypes.py +++ b/tests/test_hub/test_list_btypes.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_list_hosts.py b/tests/test_hub/test_list_hosts.py index 62f2002..f2a8aa8 100644 --- a/tests/test_hub/test_list_hosts.py +++ b/tests/test_hub/test_list_hosts.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_list_task_output.py b/tests/test_hub/test_list_task_output.py index 0b27af1..6a5f15c 100644 --- a/tests/test_hub/test_list_task_output.py +++ b/tests/test_hub/test_list_task_output.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_listing.py b/tests/test_hub/test_listing.py index a39f2b0..1647934 100644 --- a/tests/test_hub/test_listing.py +++ b/tests/test_hub/test_listing.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_models/test_host.py b/tests/test_hub/test_models/test_host.py index c1bc6e5..80f6667 100644 --- a/tests/test_hub/test_models/test_host.py +++ b/tests/test_hub/test_models/test_host.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_new_typed_build.py b/tests/test_hub/test_new_typed_build.py index a1ae47f..b7b6ccf 100644 --- a/tests/test_hub/test_new_typed_build.py +++ b/tests/test_hub/test_new_typed_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_policy_tests.py b/tests/test_hub/test_policy_tests.py index 543863a..9a3aa77 100644 --- a/tests/test_hub/test_policy_tests.py +++ b/tests/test_hub/test_policy_tests.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_query_processor.py b/tests/test_hub/test_query_processor.py index 7b229bd..e7c7cb8 100644 --- a/tests/test_hub/test_query_processor.py +++ b/tests/test_hub/test_query_processor.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_recycle_build.py b/tests/test_hub/test_recycle_build.py index 119814f..d5b81c4 100644 --- a/tests/test_hub/test_recycle_build.py +++ b/tests/test_hub/test_recycle_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import koji diff --git a/tests/test_hub/test_rpmdiff.py b/tests/test_hub/test_rpmdiff.py index 277902f..4f94f38 100644 --- a/tests/test_hub/test_rpmdiff.py +++ b/tests/test_hub/test_rpmdiff.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import unittest import mock diff --git a/tests/test_hub/test_tag_operations.py b/tests/test_hub/test_tag_operations.py index 2f0d10c..1b05efb 100644 --- a/tests/test_hub/test_tag_operations.py +++ b/tests/test_hub/test_tag_operations.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock import shutil diff --git a/tests/test_hub/test_update_processor.py b/tests/test_hub/test_update_processor.py index e765b3b..b31956a 100644 --- a/tests/test_hub/test_update_processor.py +++ b/tests/test_hub/test_update_processor.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_hub/test_user_groups.py b/tests/test_hub/test_user_groups.py index 4853560..8d551e1 100644 --- a/tests/test_hub/test_user_groups.py +++ b/tests/test_hub/test_user_groups.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_lib/test_encode_datetime.py b/tests/test_lib/test_encode_datetime.py index 1583c37..505c236 100644 --- a/tests/test_lib/test_encode_datetime.py +++ b/tests/test_lib/test_encode_datetime.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import datetime import unittest diff --git a/tests/test_lib_py2only/test_compatrequests.py b/tests/test_lib_py2only/test_compatrequests.py index 6a684ef..250cc3d 100644 --- a/tests/test_lib_py2only/test_compatrequests.py +++ b/tests/test_lib_py2only/test_compatrequests.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import six.moves.http_client import mock import unittest diff --git a/tests/test_lib_py2only/test_restart_tasks.py b/tests/test_lib_py2only/test_restart_tasks.py index bfe7b67..0517a42 100644 --- a/tests/test_lib_py2only/test_restart_tasks.py +++ b/tests/test_lib_py2only/test_restart_tasks.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import shutil import tempfile diff --git a/tests/test_plugins/load_plugin.py b/tests/test_plugins/load_plugin.py index dcc3f25..ac20795 100644 --- a/tests/test_plugins/load_plugin.py +++ b/tests/test_plugins/load_plugin.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os import sys diff --git a/tests/test_plugins/test_protonmsg.py b/tests/test_plugins/test_protonmsg.py index 61b1ec5..750ae98 100644 --- a/tests/test_plugins/test_protonmsg.py +++ b/tests/test_plugins/test_protonmsg.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest from mock import patch, MagicMock import protonmsg diff --git a/tests/test_plugins/test_runroot_hub.py b/tests/test_plugins/test_runroot_hub.py index 79b5696..4f129c8 100644 --- a/tests/test_plugins/test_runroot_hub.py +++ b/tests/test_plugins/test_runroot_hub.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest import mock diff --git a/tests/test_plugins/test_save_failed_tree_cli.py b/tests/test_plugins/test_save_failed_tree_cli.py index 1160975..d42c326 100644 --- a/tests/test_plugins/test_save_failed_tree_cli.py +++ b/tests/test_plugins/test_save_failed_tree_cli.py @@ -1,10 +1,11 @@ +from __future__ import absolute_import import mock import StringIO import unittest import koji -import load_plugin +from . import load_plugin save_failed_tree = load_plugin.load_plugin('cli', 'save_failed_tree') diff --git a/tests/test_scm.py b/tests/test_scm.py index 5742026..3628f14 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_www/test_util.py b/tests/test_www/test_util.py index 0ead026..8cc25b9 100644 --- a/tests/test_www/test_util.py +++ b/tests/test_www/test_util.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest from kojiweb.util import formatMode diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index a2c9a6d..7d2dd14 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -19,6 +19,7 @@ # Authors: # Mike McLean +from __future__ import absolute_import import cgi import inspect import koji From 9d28c5221eb18eb093dd690777eb72b9528c24e2 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:18:38 +0000 Subject: [PATCH 6/17] use six_string_types --- diff --git a/builder/kojid b/builder/kojid index e5f7196..44efdf7 100755 --- a/builder/kojid +++ b/builder/kojid @@ -49,6 +49,7 @@ import rpm import rpmUtils.arch import shutil import signal +import six import smtplib import socket import sys @@ -2757,7 +2758,7 @@ class ImageTask(BaseTaskHandler): self.ks.handler.repo.repoList = [] # delete whatever the ks file told us if opts.get('repo'): user_repos = opts['repo'] - if isinstance(user_repos, basestring): + if isinstance(user_repos, six.string_types): user_repos = user_repos.split(',') index = 0 for user_repo in user_repos: @@ -5482,7 +5483,7 @@ class WaitrepoTask(BaseTaskHandler): if not targets: raise koji.GenericError("No build target for tag: %s" % taginfo['name']) - if isinstance(newer_than, basestring) and newer_than.lower() == "now": + if isinstance(newer_than, six.string_types) and newer_than.lower() == "now": newer_than = start if not isinstance(newer_than, (type(None), int, long, float)): raise koji.GenericError("Invalid value for newer_than: %s" % newer_than) diff --git a/hub/kojihub.py b/hub/kojihub.py index b3a023f..bcc5804 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -46,6 +46,7 @@ import os import re import rpm import shutil +import six import stat import subprocess import sys @@ -53,7 +54,6 @@ import tarfile import tempfile import traceback import time -import types import six.moves.xmlrpc_client import zipfile @@ -863,7 +863,7 @@ def _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, tag_id = tag['id'] pkg = lookup_package(pkginfo, strict=False) if not pkg: - if not isinstance(pkginfo, basestring): + if not isinstance(pkginfo, six.string_types): raise koji.GenericError("Invalid package: %s" % pkginfo) if owner is not None: owner = get_user(owner, strict=True)['id'] @@ -1304,7 +1304,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late joins.append('LEFT OUTER JOIN rpmsigs on rpminfo.id = rpmsigs.rpm_id') if arch: data['arch'] = arch - if isinstance(arch, basestring): + if isinstance(arch, six.string_types): clauses.append('rpminfo.arch = %(arch)s') elif isinstance(arch, (list, tuple)): clauses.append('rpminfo.arch IN %(arch)s') @@ -2091,7 +2091,7 @@ def remove_host_from_channel(hostname, channel_name): def rename_channel(old, new): """Rename a channel""" context.session.assertPerm('admin') - if not isinstance(new, basestring): + if not isinstance(new, six.string_types): raise koji.GenericError("new channel name must be a string") cinfo = get_channel(old, strict=True) dup_check = get_channel(new, strict=False) @@ -2982,7 +2982,7 @@ def get_tag(tagInfo, strict=False, event=None): clauses = [eventCondition(event, table='tag_config')] if isinstance(tagInfo, (int, long)): clauses.append("tag.id = %(tagInfo)i") - elif isinstance(tagInfo, basestring): + elif isinstance(tagInfo, six.string_types): clauses.append("tag.name = %(tagInfo)s") else: raise koji.GenericError('invalid type for tagInfo: %s' % type(tagInfo)) @@ -5264,7 +5264,7 @@ class CG_Importer(object): if metadata is None: #default to looking for uploaded file metadata = 'metadata.json' - if not isinstance(metadata, (str, unicode)): + if not isinstance(metadata, six.string_types): raise koji.GenericError("Invalid metadata value: %r" % metadata) if metadata.endswith('.json'): # handle uploaded metadata @@ -5349,7 +5349,7 @@ class CG_Importer(object): datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ') owner = metadata['build'].get('owner', None) if owner: - if not isinstance(owner, basestring): + if not isinstance(owner, six.string_types): raise koji.GenericError("Invalid owner format (expected username): %s" % owner) buildinfo['owner'] = get_user(owner, strict=True)['id'] self.buildinfo = buildinfo @@ -5717,11 +5717,11 @@ def add_external_rpm(rpminfo, external_repo, strict=True): #sanity check rpminfo dtypes = ( - ('name', basestring), - ('version', basestring), - ('release', basestring), + ('name', six.string_types), + ('version', six.string_types), + ('release', six.string_types), ('epoch', (int, type(None))), - ('arch', basestring), + ('arch', six.string_types), ('payloadhash', str), ('size', int), ('buildtime', (int, long))) @@ -6788,7 +6788,7 @@ def query_history(tables=None, **kwargs): fields['creator.id = %(editor)i'] = '_created_by' fields['revoker.id = %(editor)i'] = '_revoked_by' elif arg == 'after': - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): value = datetime.datetime.fromtimestamp(value).isoformat(' ') data['after'] = value clauses.append('ev1.time > %(after)s OR ev2.time > %(after)s') @@ -6803,7 +6803,7 @@ def query_history(tables=None, **kwargs): fields[c_test] = '_created_after_event' fields[r_test] = '_revoked_after_event' elif arg == 'before': - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): value = datetime.datetime.fromtimestamp(value).isoformat(' ') data['before'] = value clauses.append('ev1.time < %(before)s OR ev2.time < %(before)s') @@ -7959,7 +7959,7 @@ def policy_get_pkg(data): if not pkginfo: #for some operations (e.g. adding a new package), the package #entry may not exist yet - if isinstance(data['package'], basestring): + if isinstance(data['package'], six.string_types): return {'id' : None, 'name' : data['package']} else: raise koji.GenericError("Invalid package: %s" % data['package']) @@ -8888,7 +8888,7 @@ class RootExports(object): # we will accept offset and size as strings to work around xmlrpc limits offset = koji.decode_int(offset) size = koji.decode_int(size) - if isinstance(md5sum, basestring): + if isinstance(md5sum, six.string_types): # this case is for backwards compatibility verify = "md5" digest = md5sum @@ -9454,7 +9454,7 @@ class RootExports(object): if before: if isinstance(before, datetime.datetime): before = calendar.timegm(before.utctimetuple()) - elif isinstance(before, (str, unicode)): + elif isinstance(before, six.string_types): before = koji.util.parseTime(before) elif isinstance(before, (int, long)): pass @@ -9464,7 +9464,7 @@ class RootExports(object): if after: if isinstance(after, datetime.datetime): after = calendar.timegm(after.utctimetuple()) - elif isinstance(after, (str, unicode)): + elif isinstance(after, six.string_types): after = koji.util.parseTime(after) elif isinstance(after, (int, long)): pass diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 5aeb583..0018143 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -553,7 +553,7 @@ def get_policy(opts, plugins): # tests can be limited to certain policies by setting a class variable for name, test in six.iteritems(tests): if hasattr(test, 'policy'): - if isinstance(test.policy, basestring): + if isinstance(test.policy, six.string_types): if pname != test.policy: continue elif pname not in test.policy: diff --git a/koji/compatrequests.py b/koji/compatrequests.py index a3e9aa2..3ef0175 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -11,6 +11,7 @@ from __future__ import absolute_import import six.moves.http_client import urlparse import urllib +import six import sys from . import ssl.SSLCommon from warnings import warn @@ -58,7 +59,7 @@ class Session(object): # Otherwise we make a new one default_port = 80 certs = {} - if isinstance(verify, basestring): + if isinstance(verify, six.string_types): certs['peer_ca_cert'] = verify if cert: certs['key_and_cert'] = cert diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 7d2dd14..7489647 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -27,6 +27,7 @@ import koji.util import logging import os.path import pprint +import six import sys import traceback @@ -397,14 +398,14 @@ class Dispatcher(object): else: # last one wins headers[key] = (name, value) - if isinstance(result, basestring): + if isinstance(result, six.string_types): headers.setdefault('content-length', ('Content-Length', str(len(result)))) headers.setdefault('content-type', ('Content-Type', 'text/html')) headers = list(headers.values()) + extra self.logger.debug("Headers:") self.logger.debug(koji.util.LazyString(pprint.pformat, [headers])) start_response(status, headers) - if isinstance(result, basestring): + if isinstance(result, six.string_types): result = [result] return result From d776d86ff84e27764c7694180ebc3ba3bd2af821 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:29:38 +0000 Subject: [PATCH 7/17] use six.moves.parse --- diff --git a/hub/kojihub.py b/hub/kojihub.py index bcc5804..570f2f0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -25,7 +25,6 @@ from __future__ import absolute_import import base64 import calendar -import urlparse import koji import koji.auth import koji.db @@ -54,14 +53,12 @@ import tarfile import tempfile import traceback import time -import six.moves.xmlrpc_client import zipfile import koji.xmlrpcplus from koji.context import context from six.moves import zip from six.moves import range -import six try: import json @@ -12671,7 +12668,7 @@ def handle_upload(environ): start = time.time() if not context.session.logged_in: raise koji.ActionNotAllowed('you must be logged-in to upload a file') - args = urlparse.parse_qs(environ.get('QUERY_STRING', ''), strict_parsing=True) + args = six.moves.urllib.parse.parse_qs(environ.get('QUERY_STRING', ''), strict_parsing=True) #XXX - already parsed by auth name = args['filename'][0] path = args.get('filepath', ('',))[0] diff --git a/koji/auth.py b/koji/auth.py index 706f6bc..7234547 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -29,7 +29,6 @@ try: except ImportError: krbV = None import koji -import urlparse #for parse_qs from .context import context from six.moves import range from six.moves import zip @@ -78,7 +77,7 @@ class Session(object): if not args: self.message = 'no session args' return - args = urlparse.parse_qs(args, strict_parsing=True) + args = six.moves.urllib.parse.parse_qs(args, strict_parsing=True) hostip = self.get_remote_ip(override=hostip) try: id = long(args['session-id'][0]) diff --git a/koji/compatrequests.py b/koji/compatrequests.py index 3ef0175..1ba8afb 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -9,7 +9,6 @@ the bits that koji needs. from __future__ import absolute_import import six.moves.http_client -import urlparse import urllib import six import sys @@ -29,7 +28,7 @@ class Session(object): def post(self, url, data=None, headers=None, stream=None, verify=None, cert=None, timeout=None): - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) if uri[3]: handler = "%s?%s" % (uri[2], uri[3]) else: diff --git a/koji/daemon.py b/koji/daemon.py index c41d2f2..8663fce 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -21,24 +21,23 @@ # Mike Bonnet from __future__ import absolute_import -import koji -import koji.tasks -import koji.xmlrpcplus -from koji.tasks import safe_rmtree -from koji.util import md5_constructor, adler32_constructor, parseStatus, dslice +import base64 +import errno +import logging import os import signal -import logging -import urlparse -from fnmatch import fnmatch -import base64 -import time +import six import sys +import time import traceback -import errno +from fnmatch import fnmatch from six.moves import range -import six +import koji +import koji.tasks +import koji.xmlrpcplus +from koji.tasks import safe_rmtree +from koji.util import md5_constructor, adler32_constructor, parseStatus, dslice def incremental_upload(session, fname, fd, path, retries=5, logger=None): if not fd: @@ -251,7 +250,7 @@ class SCM(object): # replace the scheme with http:// so that the urlparse works in all cases dummyurl = self.url.replace(scheme, 'http://', 1) - dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl) + dummyscheme, netloc, path, params, query, fragment = six.moves.urllib.parse.urlparse(dummyurl) user = None userhost = netloc.split('@') diff --git a/tests/test_lib_py2only/test_compatrequests.py b/tests/test_lib_py2only/test_compatrequests.py index 250cc3d..f71a9cf 100644 --- a/tests/test_lib_py2only/test_compatrequests.py +++ b/tests/test_lib_py2only/test_compatrequests.py @@ -1,8 +1,8 @@ from __future__ import absolute_import import six.moves.http_client import mock +import six import unittest -import urlparse import koji.compatrequests @@ -106,7 +106,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, no verify, no timeout session = koji.compatrequests.Session() url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) cnx = session.get_connection(uri, None, None, None) HTTPConnection.assert_called_once_with('www.fakedomain234234.org', 80) @@ -124,7 +124,7 @@ class TestSessionConnection(unittest.TestCase): def test_cached(self): session = koji.compatrequests.Session() url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) key = ('http', 'www.fakedomain234234.org', None, None, None) cnx = mock.MagicMock() session.connection = (key, cnx) @@ -135,7 +135,7 @@ class TestSessionConnection(unittest.TestCase): def test_badproto(self): session = koji.compatrequests.Session() url = 'nosuchproto://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) with self.assertRaises(IOError): ret = session.get_connection(uri, None, None, None) @@ -146,7 +146,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, no verify session = koji.compatrequests.Session() url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) timeout = 1701 cnx = session.get_connection(uri, None, None, 1701) @@ -160,7 +160,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, no verify session = koji.compatrequests.Session() url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) timeout = 1701 cnx = session.get_connection(uri, None, None, 1701) @@ -175,7 +175,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, no verify, no timeout session = koji.compatrequests.Session() url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) cnx = session.get_connection(uri, None, None, None) HTTPSConnection.assert_called_once_with('www.fakedomain234234.org', 443) @@ -188,7 +188,7 @@ class TestSessionConnection(unittest.TestCase): # no verify, no timeout session = koji.compatrequests.Session() url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) cert = '/path/to/cert/file' context = mock.MagicMock() CreateSSLContext.return_value = context @@ -205,7 +205,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, verify=False, no timeout session = koji.compatrequests.Session() url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) context = mock.MagicMock() create_unverified_context.return_value = context @@ -221,7 +221,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, verify=False, no timeout session = koji.compatrequests.Session() url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) cnx = session.get_connection(uri, None, False, None) HTTPSConnection.assert_called_once_with('www.fakedomain234234.org', 443) @@ -236,7 +236,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, no timeout session = koji.compatrequests.Session() url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) context = mock.MagicMock() SSLContext.return_value = context verify = '/path/to/verify/cert' @@ -257,7 +257,7 @@ class TestSessionConnection(unittest.TestCase): # no cert, no timeout session = koji.compatrequests.Session() url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2' - uri = urlparse.urlsplit(url) + uri = six.moves.urllib.parse.urlsplit(url) verify = '/path/to/verify/cert' cnx = session.get_connection(uri, None, verify, None) From 49ee498cd54ef261d991c843b13f7b2d2111b3d9 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:31:26 +0000 Subject: [PATCH 8/17] use six.StringIO --- diff --git a/tests/test_plugins/test_protonmsg.py b/tests/test_plugins/test_protonmsg.py index 750ae98..5b88dc8 100644 --- a/tests/test_plugins/test_protonmsg.py +++ b/tests/test_plugins/test_protonmsg.py @@ -3,8 +3,8 @@ import unittest from mock import patch, MagicMock import protonmsg from koji.context import context +import six import tempfile -from StringIO import StringIO from six.moves.configparser import SafeConfigParser class TestProtonMsg(unittest.TestCase): @@ -250,7 +250,7 @@ test_mode = on class TestTimeoutHandler(unittest.TestCase): def setUp(self): - confdata = StringIO("""[broker] + confdata = six.StringIO("""[broker] urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671 cert = /etc/koji-hub/plugins/client.pem cacert = /etc/koji-hub/plugins/ca.pem @@ -273,7 +273,7 @@ send_timeout = 60 @patch('protonmsg.SSLDomain') def test_on_start_no_ssl(self, SSLDomain): - confdata = StringIO("""[broker] + confdata = six.StringIO("""[broker] urls = amqp://broker1.example.com:5672 amqp://broker2.example.com:5672 topic_prefix = koji connect_timeout = 10 From 8e758106b451273cbe41ad6496c303948f5ba7df Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:31:43 +0000 Subject: [PATCH 9/17] Remove testing dir also within failing test --- diff --git a/tests/test_hub/test_get_upload_path.py b/tests/test_hub/test_get_upload_path.py index d11e1b1..4642b00 100644 --- a/tests/test_hub/test_get_upload_path.py +++ b/tests/test_hub/test_get_upload_path.py @@ -8,7 +8,11 @@ from koji import GenericError class TestGetUploadPath(unittest.TestCase): - + def tearDown(self): + try: + shutil.rmtree('/tmp/fake') + except Exception: + pass def test_get_upload_path_invalid_filename(self): with self.assertRaises(GenericError): @@ -43,8 +47,6 @@ class TestGetUploadPath(unittest.TestCase): with self.assertRaises(GenericError): kojihub.get_upload_path(reldir=reldir, name='error', create=True) - shutil.rmtree('/tmp/fake') - @mock.patch('koji.pathinfo.work') @mock.patch('kojihub.Host') def test_get_upload_path_invalid_upload_no_dir_owner(self, host, work): From 7d014a31c8346dfdcbf1df70683782feeac769e2 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:33:20 +0000 Subject: [PATCH 10/17] remove unused imports --- diff --git a/util/koji-shadow b/util/koji-shadow index 12f9997..a958c9c 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -31,7 +31,6 @@ import ConfigParser import fnmatch import optparse import os -import pprint import random import shutil import socket # for socket.error and socket.setdefaulttimeout @@ -39,7 +38,6 @@ import string import sys import time import urllib2 -import urlgrabber.grabber as grabber import xmlrpclib # for ProtocolError and Fault import rpm From 94be268544314ee22fcf8b323b05b0498bcdcdcd Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:35:25 +0000 Subject: [PATCH 11/17] make loadkojid/loadcli working in python3 --- diff --git a/tests/test_builder/loadkojid.py b/tests/test_builder/loadkojid.py index d4e26e5..e469ed4 100644 --- a/tests/test_builder/loadkojid.py +++ b/tests/test_builder/loadkojid.py @@ -6,9 +6,12 @@ import sys KOJID_FILENAME = os.path.dirname(__file__) + "/../../builder/kojid" if sys.version_info[0] >= 3: import importlib.util - spec = importlib.util.spec_from_file_location("koji_kojid", KOJID_FILENAME) + import importlib.machinery + loader = importlib.machinery.SourceFileLoader('kojid_kojid', KOJID_FILENAME) + spec = importlib.util.spec_from_loader(loader.name, loader) kojid = importlib.util.module_from_spec(spec) - spec.loader.exec_module(kojid) + loader.exec_module(kojid) + sys.modules['kojid_kojid'] = kojid else: import imp kojid = imp.load_source('koji_kojid', KOJID_FILENAME) diff --git a/tests/test_cli/loadcli.py b/tests/test_cli/loadcli.py index c2800ac..dc2a6fa 100644 --- a/tests/test_cli/loadcli.py +++ b/tests/test_cli/loadcli.py @@ -1,19 +1,19 @@ from __future__ import absolute_import import os -#import sys +import sys # We have to do this craziness because 'import koji' is ambiguous. Is it the # koji module, or the koji cli module. Jump through hoops accordingly. # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path CLI_FILENAME = os.path.dirname(__file__) + "/../../cli/koji" -''' if sys.version_info[0] >= 3: import importlib.util - spec = importlib.util.spec_from_file_location("koji_cli", CLI_FILENAME) + import importlib.machinery + loader = importlib.machinery.SourceFileLoader('koji_cli', CLI_FILENAME) + spec = importlib.util.spec_from_loader(loader.name, loader) cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + loader.exec_module(cli) + sys.modules['koji_cli_fake'] = cli else: -''' - -import imp -cli = imp.load_source('koji_cli_fake', CLI_FILENAME) + import imp + cli = imp.load_source('koji_cli_fake', CLI_FILENAME) diff --git a/tests/test_plugins/load_plugin.py b/tests/test_plugins/load_plugin.py index ac20795..0e68dbc 100644 --- a/tests/test_plugins/load_plugin.py +++ b/tests/test_plugins/load_plugin.py @@ -7,23 +7,23 @@ def load_plugin(plugin_type, plugin_name): # koji module, or the koji cli module. Jump through hoops accordingly. # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path mod_name = "%s_%s" % (plugin_name, plugin_type) - CLI_FILENAME = os.path.join( + PLUGIN_FILENAME = os.path.join( os.path.dirname(__file__), "../../plugins", plugin_type, "%s.py" % plugin_name) - sys.path = [os.path.dirname(CLI_FILENAME), + sys.path = [os.path.dirname(PLUGIN_FILENAME), os.path.join(os.path.dirname(__file__), "../..", plugin_type)] + \ sys.path if sys.version_info[0] >= 3: import importlib.machinery - loader = importlib.machinery.SourceFileLoader(mod_name, CLI_FILENAME) + loader = importlib.machinery.SourceFileLoader(mod_name, PLUGIN_FILENAME) spec = importlib.util.spec_from_loader(loader.name, loader) - kojid = importlib.util.module_from_spec(spec) - spec.loader.exec_module(kojid) - loader.exec_module(kojid) - sys.modules[mod_name] = kojid + plugin = importlib.util.module_from_spec(spec) + spec.loader.exec_module(plugin) + loader.exec_module(plugin) + sys.modules[mod_name] = plugin else: import imp - plugin = imp.load_source(mod_name, CLI_FILENAME) + plugin = imp.load_source(mod_name, PLUGIN_FILENAME) return plugin diff --git a/tests/test_plugins/test_save_failed_tree_cli.py b/tests/test_plugins/test_save_failed_tree_cli.py index d42c326..bb7b1d1 100644 --- a/tests/test_plugins/test_save_failed_tree_cli.py +++ b/tests/test_plugins/test_save_failed_tree_cli.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import mock -import StringIO +from six import StringIO import unittest import koji @@ -121,7 +121,7 @@ class TestSaveFailedTree(unittest.TestCase): poll_interval=options.poll_interval, quiet=options.quiet) - @mock.patch('sys.stdout', new_callable=StringIO.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_handle_save_failed_tree_errors(self, stdout): # koji save-failed-tree 123 456 arguments = [123, 456] From a015c8e9c7779a6103d6112ebcb697ec28354669 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:36:17 +0000 Subject: [PATCH 12/17] make sql generation deterministic --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 570f2f0..1952a8e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -7533,8 +7533,8 @@ class InsertProcessor(object): if not self.data and not self.rawdata: return "-- incomplete update: no assigns" parts = ['INSERT INTO %s ' % self.table] - columns = list(self.data.keys()) - columns.extend(list(self.rawdata.keys())) + columns = sorted(self.data.keys()) + columns.extend(sorted(self.rawdata.keys())) parts.append("(%s) " % ', '.join(columns)) values = [] for key in columns: @@ -7762,7 +7762,7 @@ SELECT %(col_str)s def _seqtostr(self, seq, sep=', '): if seq: - return sep.join(seq) + return sep.join(sorted(seq)) else: return '' diff --git a/koji/tasks.py b/koji/tasks.py index 10a74d9..1a3e5c9 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -373,7 +373,7 @@ class BaseTaskHandler(object): else: # no overlap raise koji.BuildError("host %s (%s) does not support any arches of tag %s (%s)" % \ - (host['name'], ', '.join(host_arches), tag['name'], ', '.join(tag_arches))) + (host['name'], ', '.join(sorted(host_arches)), tag['name'], ', '.join(sorted(tag_arches)))) def getRepo(self, tag): """ diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index 1c329d9..706bb3e 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -87,8 +87,8 @@ class TestImportRPM(unittest.TestCase): 'size', ] statement = 'INSERT INTO rpminfo (%s) VALUES (%s)' % ( - ", ".join(fields), - ", ".join(['%%(%s)s' % field for field in fields]) + ", ".join(sorted(fields)), + ", ".join(['%%(%s)s' % field for field in sorted(fields)]) ) values = { 'build_id': 12345, @@ -146,8 +146,8 @@ class TestImportRPM(unittest.TestCase): 'size', ] statement = 'INSERT INTO rpminfo (%s) VALUES (%s)' % ( - ", ".join(fields), - ", ".join(['%%(%s)s' % field for field in fields]) + ", ".join(sorted(fields)), + ", ".join(['%%(%s)s' % field for field in sorted(fields)]) ) values = { 'build_id': 12345, @@ -264,8 +264,8 @@ class TestImportBuild(unittest.TestCase): 'id', ] statement = 'INSERT INTO build (%s) VALUES (%s)' % ( - ", ".join(fields), - ", ".join(['%%(%s)s' % field for field in fields]) + ", ".join(sorted(fields)), + ", ".join(['%%(%s)s' % field for field in sorted(fields)]) ) values = { 'task_id': None, From 44a5953186490767f98c783016a526c6ed100c7f Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:37:22 +0000 Subject: [PATCH 13/17] fix hub py3 compatibility --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 1952a8e..6884c55 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -587,7 +587,7 @@ def eventCondition(event, table=None): table += '.' if event is None: return """(%(table)sactive = TRUE)""" % locals() - elif isinstance(event, int) or isinstance(event, long): + elif isinstance(event, six.integer_types): return """(%(table)screate_event <= %(event)d AND ( %(table)srevoke_event IS NULL OR %(event)d < %(table)srevoke_event ))""" \ % locals() else: @@ -1030,7 +1030,7 @@ def readPackageList(tagID=None, userID=None, pkgID=None, event=None, inherit=Fal q += """ AND users.id = %%(userID)i""" if pkgID != None: - if isinstance(pkgID, int) or isinstance(pkgID, long): + if isinstance(pkgID, six.integer_types): q += """ AND package.id = %%(pkgID)i""" else: @@ -1159,7 +1159,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, # build - id pkg_id version release epoch # tag_listing - id build_id tag_id - if not isinstance(latest, (int, long, float)): + if not isinstance(latest, six.integer_types + (float,)): latest = bool(latest) taglist = [tag] @@ -2755,7 +2755,7 @@ def get_build_targets(info=None, event=None, buildTagID=None, destTagID=None, qu if info: if isinstance(info, str): clauses.append('build_target.name = %(info)s') - elif isinstance(info, int) or isinstance(info, long): + elif isinstance(info, six.integer_types): clauses.append('build_target.id = %(info)i') else: raise koji.GenericError('invalid type for lookup: %s' % type(info)) @@ -2798,7 +2798,7 @@ def lookup_name(table, info, strict=False, create=False): create option will fail. """ fields = ('id', 'name') - if isinstance(info, int) or isinstance(info, long): + if isinstance(info, six.integer_types): q = """SELECT id,name FROM %s WHERE id=%%(info)d""" % table elif isinstance(info, str): q = """SELECT id,name FROM %s WHERE name=%%(info)s""" % table @@ -2977,7 +2977,7 @@ def get_tag(tagInfo, strict=False, event=None): 'tag_config.maven_include_all': 'maven_include_all' } clauses = [eventCondition(event, table='tag_config')] - if isinstance(tagInfo, (int, long)): + if isinstance(tagInfo, six.integer_types): clauses.append("tag.id = %(tagInfo)i") elif isinstance(tagInfo, six.string_types): clauses.append("tag.name = %(tagInfo)s") @@ -3205,7 +3205,7 @@ def get_external_repos(info=None, url=None, event=None, queryOpts=None): if info is not None: if isinstance(info, str): clauses.append('name = %(info)s') - elif isinstance(info, (int, long)): + elif isinstance(info, six.integer_types): clauses.append('id = %(info)i') else: raise koji.GenericError('invalid type for lookup: %s' % type(info)) @@ -3430,7 +3430,7 @@ def get_user(userInfo=None, strict=False): fields = ['id', 'name', 'status', 'usertype', 'krb_principal'] #fields, aliases = zip(*fields.items()) data = {'info' : userInfo} - if isinstance(userInfo, int) or isinstance(userInfo, long): + if isinstance(userInfo, six.integer_types): clauses = ['id = %(info)i'] elif isinstance(userInfo, str): clauses = ['krb_principal = %(info)s OR name = %(info)s'] @@ -3445,7 +3445,7 @@ def get_user(userInfo=None, strict=False): def find_build_id(X, strict=False): - if isinstance(X, int) or isinstance(X, long): + if isinstance(X, six.integer_types): return X elif isinstance(X, str): data = koji.parse_NVR(X) @@ -3676,7 +3676,7 @@ def get_rpm(rpminfo, strict=False, multi=False): ) # we can look up by id or NVRA data = None - if isinstance(rpminfo, (int, long)): + if isinstance(rpminfo, six.integer_types): data = {'id': rpminfo} elif isinstance(rpminfo, str): data = koji.parse_NVRA(rpminfo) @@ -4501,7 +4501,7 @@ def get_host(hostInfo, strict=False): 'capacity', 'description', 'comment', 'ready', 'enabled') query = """SELECT %s FROM host WHERE """ % ', '.join(fields) - if isinstance(hostInfo, int) or isinstance(hostInfo, long): + if isinstance(hostInfo, six.integer_types): query += """id = %(hostInfo)i""" elif isinstance(hostInfo, str): query += """name = %(hostInfo)s""" @@ -4550,7 +4550,7 @@ def get_channel(channelInfo, strict=False): fields = ('id', 'name') query = """SELECT %s FROM channels WHERE """ % ', '.join(fields) - if isinstance(channelInfo, int) or isinstance(channelInfo, long): + if isinstance(channelInfo, six.integer_types): query += """id = %(channelInfo)i""" elif isinstance(channelInfo, str): query += """name = %(channelInfo)s""" @@ -5269,7 +5269,7 @@ class CG_Importer(object): path = os.path.join(workdir, directory, metadata) if not os.path.exists(path): raise koji.GenericError("No such file: %s" % metadata) - fo = open(path, 'rb') + fo = open(path, 'r') metadata = fo.read() fo.close() self.raw_metadata = metadata @@ -5721,7 +5721,7 @@ def add_external_rpm(rpminfo, external_repo, strict=True): ('arch', six.string_types), ('payloadhash', str), ('size', int), - ('buildtime', (int, long))) + ('buildtime', six.integer_types)) for field, allowed in dtypes: if field not in rpminfo: raise koji.GenericError("%s field missing: %r" % (field, rpminfo)) @@ -7495,7 +7495,9 @@ def parse_json(value, desc=None, errstr=None): if value is None: return value try: - return koji.fixEncodingRecurse(json.loads(value)) + #return koji.fixEncodingRecurse(json.loads(value)) + # JSON has to be utf-8, so no need for fixEncodingRecurse + return json.loads(value) except Exception: if errstr is None: if desc is None: @@ -8853,7 +8855,7 @@ class RootExports(object): values = {} q = """SELECT id, EXTRACT(EPOCH FROM time) FROM events""" if before is not None: - if not isinstance(before, (int, long, float)): + if not isinstance(before, six.integer_types + (float,)): raise koji.GenericError('invalid type for before: %s' % type(before)) # use the repr() conversion because it retains more precision than the # string conversion @@ -9453,7 +9455,7 @@ class RootExports(object): before = calendar.timegm(before.utctimetuple()) elif isinstance(before, six.string_types): before = koji.util.parseTime(before) - elif isinstance(before, (int, long)): + elif isinstance(before, six.integer_types): pass else: raise koji.GenericError('invalid type for before: %s' % type(before)) @@ -9463,7 +9465,7 @@ class RootExports(object): after = calendar.timegm(after.utctimetuple()) elif isinstance(after, six.string_types): after = koji.util.parseTime(after) - elif isinstance(after, (int, long)): + elif isinstance(after, six.integer_types): pass else: raise koji.GenericError('invalid type for after: %s' % type(after)) @@ -9552,7 +9554,7 @@ class RootExports(object): def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, owner=None, type=None): """List builds tagged with tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) results = readTaggedBuilds(tag, event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) @@ -9563,14 +9565,14 @@ class RootExports(object): def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, rpmsigs=False, owner=None, type=None): """List rpms and builds within tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, event=event, inherit=inherit, latest=latest, package=package, arch=arch, rpmsigs=rpmsigs, owner=owner, type=type) def listTaggedArchives(self, tag, event=None, inherit=False, latest=False, package=None, type=None): """List archives and builds within a tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): tag = get_tag_id(tag, strict=True) return readTaggedArchives(tag, event=event, inherit=inherit, latest=latest, package=package, type=type) @@ -9727,14 +9729,14 @@ class RootExports(object): def getLatestBuilds(self, tag, event=None, package=None, type=None): """List latest builds for tag (inheritance enabled)""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type) def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None): """List latest RPMS for tag (inheritance enabled)""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, latest=True, rpmsigs=rpmsigs, type=type) @@ -9794,13 +9796,13 @@ class RootExports(object): def getInheritanceData(self, tag, event=None): """Return inheritance data for tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readInheritanceData(tag, event) def setInheritanceData(self, tag, data, clear=False): - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) context.session.assertPerm('admin') @@ -9811,7 +9813,7 @@ class RootExports(object): stops = {} if jumps is None: jumps = {} - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) for mapping in [stops, jumps]: @@ -9838,7 +9840,7 @@ class RootExports(object): - buildroot_id If no build has the given ID, or the build generated no RPMs, an empty list is returned.""" - if not isinstance(build, (int, long)): + if not isinstance(build, six.integer_types): #lookup build id build = self.findBuildID(build, strict=True) return self.listRPMs(buildID=build) @@ -10225,7 +10227,7 @@ class RootExports(object): return taginfo def getRepo(self, tag, state=None, event=None, dist=False): - if isinstance(tag, (int, long)): + if isinstance(tag, six.integer_types): id = tag else: id = get_tag_id(tag, strict=True) @@ -10742,7 +10744,7 @@ class RootExports(object): ts = time.mktime(time.strptime(str(ts), '%Y%m%dT%H:%M:%S')) except ValueError: raise koji.GenericError("Invalid time: %s" % ts) - elif not isinstance(ts, (int, long, float)): + elif not isinstance(ts, six.integer_types + (float,)): raise koji.GenericError("Invalid type for timestamp") koji.plugin.run_callbacks('preBuildStateChange', attribute='completion_ts', old=buildinfo['completion_ts'], new=ts, info=buildinfo) buildid = buildinfo['id'] @@ -12299,7 +12301,7 @@ class HostExports(object): extra_deps = [] task_deps = {} for dep in extra_deps: - if isinstance(dep, (int, long)): + if isinstance(dep, six.integer_types): task_output = list_task_output(dep, stat=True) for filepath, filestats in six.iteritems(task_output): if os.path.splitext(filepath)[1] in ['.log', '.md5', '.sha1']: diff --git a/tests/test_hub/cg_importer_json/default.json b/tests/test_hub/cg_importer_json/default.json index 03eb402..2c54276 100644 --- a/tests/test_hub/cg_importer_json/default.json +++ b/tests/test_hub/cg_importer_json/default.json @@ -77,7 +77,7 @@ "repositories": ["repository.example.com/username/imagename:tagname", "repository.example.com/username/imagename:latest"]}}}, {"buildroot_id": 1, - "filename": "checkout.log", + "filename": "tady_checkoutěščřž.log", "filesize": 85724, "arch": "noarch", "checksum_type": "md5", diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 708a464..72f659e 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -4,7 +4,6 @@ import mock import os import shutil -import koji import kojihub from koji import GenericError diff --git a/tests/test_hub/test_getRPMDeps.py b/tests/test_hub/test_getRPMDeps.py index f4931a4..db9cf3d 100644 --- a/tests/test_hub/test_getRPMDeps.py +++ b/tests/test_hub/test_getRPMDeps.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import mock import os +import six import unittest import koji @@ -34,14 +35,14 @@ class TestGetRPMDeps(unittest.TestCase): result = {} for r in res: result.setdefault(r['type'], set()).add(r['name']) - self.assertTrue('require1' in result[koji.DEP_REQUIRE]) - self.assertTrue('provide1' in result[koji.DEP_PROVIDE]) - self.assertTrue('obsoletes1' in result[koji.DEP_OBSOLETE]) - self.assertTrue('conflicts1' in result[koji.DEP_CONFLICT]) - self.assertTrue('suggests1' in result[koji.DEP_SUGGEST]) - self.assertTrue('enhances1' in result[koji.DEP_ENHANCE]) - self.assertTrue('supplements1' in result[koji.DEP_SUPPLEMENT]) - self.assertTrue('recommends1' in result[koji.DEP_RECOMMEND]) + self.assertTrue(six.b('require1') in result[koji.DEP_REQUIRE]) + self.assertTrue(six.b('provide1') in result[koji.DEP_PROVIDE]) + self.assertTrue(six.b('obsoletes1') in result[koji.DEP_OBSOLETE]) + self.assertTrue(six.b('conflicts1') in result[koji.DEP_CONFLICT]) + self.assertTrue(six.b('suggests1') in result[koji.DEP_SUGGEST]) + self.assertTrue(six.b('enhances1') in result[koji.DEP_ENHANCE]) + self.assertTrue(six.b('supplements1') in result[koji.DEP_SUPPLEMENT]) + self.assertTrue(six.b('recommends1') in result[koji.DEP_RECOMMEND]) else: self.assertEqual(len(res), 14) types = set([x['type'] for x in res]) diff --git a/tests/test_hub/test_get_upload_path.py b/tests/test_hub/test_get_upload_path.py index 4642b00..fb6b35c 100644 --- a/tests/test_hub/test_get_upload_path.py +++ b/tests/test_hub/test_get_upload_path.py @@ -41,7 +41,7 @@ class TestGetUploadPath(unittest.TestCase): fullpath = '{0}/{1}'.format(work.return_value, reldir) os.makedirs(fullpath) - with open('{0}/.user'.format(fullpath), 'wb') as f: + with open('{0}/.user'.format(fullpath), 'w') as f: f.write('1') with self.assertRaises(GenericError): From 0cb09fb13ccaf35860f3812289c1d2eec12e0a7f Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:37:32 +0000 Subject: [PATCH 14/17] add test_hub to test3 target --- diff --git a/.coveragerc3 b/.coveragerc3 index 236173b..5b4ae73 100644 --- a/.coveragerc3 +++ b/.coveragerc3 @@ -5,8 +5,5 @@ omit = /usr/* tests/* - hub/* util/* koji/ssl/* - koji/daemon.py - koji/tasks.py diff --git a/Makefile b/Makefile index 51e9e95..6419e3f 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ test3: PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/. coverage3 run \ --rcfile .coveragerc3 --source . \ /usr/bin/nosetests-3 \ - tests/test_lib tests/test_cli + tests/test_lib tests/test_cli tests/test_hub tests/*.py coverage report coverage html @echo Full coverage report in htmlcov/index.html From 29856c3bd0be7692908724bfd51b28d54b42d0a3 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 15:37:40 +0000 Subject: [PATCH 15/17] missing import for mocking test --- diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 72f659e..708a464 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -4,6 +4,7 @@ import mock import os import shutil +import koji import kojihub from koji import GenericError From 751f840bd7669271cf0f614d06f1aa5feec4e61f Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 16:44:48 +0000 Subject: [PATCH 16/17] update to latest master --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 6884c55..552cd17 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -6233,7 +6233,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No filename = koji.fixEncoding(os.path.basename(filepath)) archiveinfo['filename'] = filename archiveinfo['size'] = os.path.getsize(filepath) - archivefp = open(filepath) + archivefp = open(filepath, 'rb') m = md5_constructor() while True: contents = archivefp.read(8192) @@ -6348,8 +6348,8 @@ def _import_archive_file(filepath, destdir): A symlink pointing from the old location to the new location will be created. """ - final_path = "%s/%s" % (destdir, - koji.fixEncoding(os.path.basename(filepath))) + final_path = os.path.join(destdir, + koji.fixEncoding(os.path.basename(filepath))) if os.path.exists(final_path): raise koji.GenericError("Error importing archive file, %s already exists" % final_path) if os.path.islink(filepath) or not os.path.isfile(filepath): @@ -6373,7 +6373,7 @@ def _generate_maven_metadata(mavendir): sumfile = mavenfile + ext if sumfile not in mavenfiles: sum = sum_constr() - fobj = open('%s/%s' % (mavendir, mavenfile)) + fobj = open('%s/%s' % (mavendir, mavenfile), 'rb') while True: content = fobj.read(8192) if not content: @@ -7620,8 +7620,8 @@ class UpdateProcessor(object): if not self.data and not self.rawdata: return "-- incomplete update: no assigns" parts = ['UPDATE %s SET ' % self.table] - assigns = ["%s = %%(data.%s)s" % (key, key) for key in self.data] - assigns.extend(["%s = (%s)" % (key, self.rawdata[key]) for key in self.rawdata]) + assigns = ["%s = %%(data.%s)s" % (key, key) for key in sorted(self.data)] + assigns.extend(["%s = (%s)" % (key, self.rawdata[key]) for key in sorted(self.rawdata)]) parts.append(', '.join(assigns)) if self.clauses: parts.append('\nWHERE ') @@ -8500,7 +8500,7 @@ def importImageInternal(task_id, build_id, imgdata): insert = InsertProcessor('archive_rpm_components') for archive in archives: logger.info('working on archive %s', archive) - if archive['filename'].endswith('xml'): + if archive['filename'].endswith('.xml'): continue insert.set(archive_id = archive['id']) logger.info('associating installed rpms with %s', archive['id']) diff --git a/koji/compatrequests.py b/koji/compatrequests.py index 1ba8afb..8e47f14 100644 --- a/koji/compatrequests.py +++ b/koji/compatrequests.py @@ -12,7 +12,7 @@ import six.moves.http_client import urllib import six import sys -from . import ssl.SSLCommon +from .ssl import SSLCommon from warnings import warn try: from .ssl import ssl as pyssl @@ -62,9 +62,9 @@ class Session(object): certs['peer_ca_cert'] = verify if cert: certs['key_and_cert'] = cert - ctx = ssl.SSLCommon.CreateSSLContext(certs) + ctx = SSLCommon.CreateSSLContext(certs) cnxOpts = {'ssl_context' : ctx} - cnxClass = ssl.SSLCommon.PlgHTTPSConnection + cnxClass = SSLCommon.PlgHTTPSConnection default_port = 443 elif scheme == 'https': cnxOpts = {} diff --git a/tests/test_docs_version.py b/tests/test_docs_version.py index 03711a1..055b21c 100644 --- a/tests/test_docs_version.py +++ b/tests/test_docs_version.py @@ -1,5 +1,6 @@ from __future__ import absolute_import import os +import six import subprocess import unittest @@ -22,7 +23,10 @@ class TestDocsVersion(unittest.TestCase): output = subprocess.check_output(cmd) # rpm outputs a line for each subpackage version = output.splitlines()[0] - return version + if six.PY2: + return version + else: + return version.decode() def test_docs_version(self): koji_version = self.get_koji_version() diff --git a/tests/test_hub/data/image/import_1/db.json b/tests/test_hub/data/image/import_1/db.json index 9ffdfb2..29ee9cd 100644 --- a/tests/test_hub/data/image/import_1/db.json +++ b/tests/test_hub/data/image/import_1/db.json @@ -1,7 +1,7 @@ { "inserts": [ [ - "INSERT INTO archiveinfo (build_id, archive_id, type_id, checksum, filename, checksum_type, btype_id, buildroot_id, id, size) VALUES (%(build_id)s, %(archive_id)s, %(type_id)s, %(checksum)s, %(filename)s, %(checksum_type)s, %(btype_id)s, %(buildroot_id)s, %(id)s, %(size)s)", + "INSERT INTO archiveinfo (archive_id, btype_id, build_id, buildroot_id, checksum, checksum_type, filename, id, size, type_id) VALUES (%(archive_id)s, %(btype_id)s, %(build_id)s, %(buildroot_id)s, %(checksum)s, %(checksum_type)s, %(filename)s, %(id)s, %(size)s, %(type_id)s)", { "build_id": 137, "archive_id": "ARCHIVE_ID", @@ -17,7 +17,7 @@ {} ], [ - "INSERT INTO image_archives (archive_id, arch) VALUES (%(archive_id)s, %(arch)s)", + "INSERT INTO image_archives (arch, archive_id) VALUES (%(arch)s, %(archive_id)s)", { "archive_id": 1001, "arch": "x86_64" @@ -25,7 +25,7 @@ {} ], [ - "INSERT INTO archiveinfo (build_id, archive_id, type_id, checksum, filename, checksum_type, btype_id, buildroot_id, id, size) VALUES (%(build_id)s, %(archive_id)s, %(type_id)s, %(checksum)s, %(filename)s, %(checksum_type)s, %(btype_id)s, %(buildroot_id)s, %(id)s, %(size)s)", + "INSERT INTO archiveinfo (archive_id, btype_id, build_id, buildroot_id, checksum, checksum_type, filename, id, size, type_id) VALUES (%(archive_id)s, %(btype_id)s, %(build_id)s, %(buildroot_id)s, %(checksum)s, %(checksum_type)s, %(filename)s, %(id)s, %(size)s, %(type_id)s)", { "build_id": 137, "archive_id": "ARCHIVE_ID", @@ -41,7 +41,7 @@ {} ], [ - "INSERT INTO image_archives (archive_id, arch) VALUES (%(archive_id)s, %(arch)s)", + "INSERT INTO image_archives (arch, archive_id) VALUES (%(arch)s, %(archive_id)s)", { "archive_id": 1002, "arch": "x86_64" @@ -49,7 +49,7 @@ {} ], [ - "INSERT INTO archiveinfo (build_id, archive_id, type_id, checksum, filename, checksum_type, btype_id, buildroot_id, id, size) VALUES (%(build_id)s, %(archive_id)s, %(type_id)s, %(checksum)s, %(filename)s, %(checksum_type)s, %(btype_id)s, %(buildroot_id)s, %(id)s, %(size)s)", + "INSERT INTO archiveinfo (archive_id, btype_id, build_id, buildroot_id, checksum, checksum_type, filename, id, size, type_id) VALUES (%(archive_id)s, %(btype_id)s, %(build_id)s, %(buildroot_id)s, %(checksum)s, %(checksum_type)s, %(filename)s, %(id)s, %(size)s, %(type_id)s)", { "build_id": 137, "archive_id": "ARCHIVE_ID", @@ -65,7 +65,7 @@ {} ], [ - "INSERT INTO image_archives (archive_id, arch) VALUES (%(archive_id)s, %(arch)s)", + "INSERT INTO image_archives (arch, archive_id) VALUES (%(arch)s, %(archive_id)s)", { "archive_id": 1003, "arch": "x86_64" @@ -73,7 +73,7 @@ {} ], [ - "INSERT INTO archiveinfo (build_id, archive_id, type_id, checksum, filename, checksum_type, btype_id, buildroot_id, id, size) VALUES (%(build_id)s, %(archive_id)s, %(type_id)s, %(checksum)s, %(filename)s, %(checksum_type)s, %(btype_id)s, %(buildroot_id)s, %(id)s, %(size)s)", + "INSERT INTO archiveinfo (archive_id, btype_id, build_id, buildroot_id, checksum, checksum_type, filename, id, size, type_id) VALUES (%(archive_id)s, %(btype_id)s, %(build_id)s, %(buildroot_id)s, %(checksum)s, %(checksum_type)s, %(filename)s, %(id)s, %(size)s, %(type_id)s)", { "build_id": 137, "archive_id": "ARCHIVE_ID", @@ -89,7 +89,7 @@ {} ], [ - "INSERT INTO image_archives (archive_id, arch) VALUES (%(archive_id)s, %(arch)s)", + "INSERT INTO image_archives (arch, archive_id) VALUES (%(arch)s, %(archive_id)s)", { "archive_id": 1004, "arch": "x86_64" @@ -97,7 +97,7 @@ {} ], [ - "INSERT INTO archiveinfo (build_id, archive_id, type_id, checksum, filename, checksum_type, btype_id, buildroot_id, id, size) VALUES (%(build_id)s, %(archive_id)s, %(type_id)s, %(checksum)s, %(filename)s, %(checksum_type)s, %(btype_id)s, %(buildroot_id)s, %(id)s, %(size)s)", + "INSERT INTO archiveinfo (archive_id, btype_id, build_id, buildroot_id, checksum, checksum_type, filename, id, size, type_id) VALUES (%(archive_id)s, %(btype_id)s, %(build_id)s, %(buildroot_id)s, %(checksum)s, %(checksum_type)s, %(filename)s, %(id)s, %(size)s, %(type_id)s)", { "build_id": 137, "archive_id": "ARCHIVE_ID", @@ -113,7 +113,7 @@ {} ], [ - "INSERT INTO image_archives (archive_id, arch) VALUES (%(archive_id)s, %(arch)s)", + "INSERT INTO image_archives (arch, archive_id) VALUES (%(arch)s, %(archive_id)s)", { "archive_id": 1005, "arch": "x86_64" @@ -195,7 +195,7 @@ ], "updates": [ [ - "UPDATE build SET state = %(data.state)s, id = %(data.id)s, completion_time = (now())\nWHERE ( id=%(build_id)i )", + "UPDATE build SET id = %(data.id)s, state = %(data.state)s, completion_time = (now())\nWHERE ( id=%(build_id)i )", { "state": 1, "id": "BUILD_ID" diff --git a/tests/test_hub/test_complete_image_build.py b/tests/test_hub/test_complete_image_build.py index 2742afd..bdadb2f 100644 --- a/tests/test_hub/test_complete_image_build.py +++ b/tests/test_hub/test_complete_image_build.py @@ -5,6 +5,7 @@ import mock import os import os.path import shutil +import six import tempfile import unittest @@ -269,4 +270,5 @@ class TestCompleteImageBuild(unittest.TestCase): # with our other mocks, we should never reach _dml self._dml.assert_not_called() data = {'inserts': self.inserts, 'updates': self.updates} + self.maxDiff = None self.assertEqual(data, self.db_expect) diff --git a/tests/test_hub/test_get_build_logs.py b/tests/test_hub/test_get_build_logs.py index 8add61f..e33054c 100644 --- a/tests/test_hub/test_get_build_logs.py +++ b/tests/test_hub/test_get_build_logs.py @@ -14,6 +14,7 @@ class TestGetBuildLogs(unittest.TestCase): def setUp(self): self.get_build = mock.patch('kojihub.get_build').start() self.pathinfo = mock.patch('koji.pathinfo').start() + self.pathinfo.topdir = 'topdir' self.tempdir = tempfile.mkdtemp() koji.pathinfo.build_logs.return_value = self.tempdir diff --git a/tests/test_scm.py b/tests/test_scm.py index 3628f14..b7dad5b 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -208,8 +208,8 @@ class TestSCMCheckouts(unittest.TestCase): self.log_output.return_value = None self.tempdir = tempfile.mkdtemp() self.session = mock.MagicMock() - self.uploadpath = mock.MagicMock() - self.logfile = mock.MagicMock() + self.uploadpath = 'uploadpath' + self.logfile = 'logfile' self.config = ''' default:* nocommon:*:no From 6beac30dff9a7e6668e7fa912941e26c254e32d5 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 07 2017 16:46:24 +0000 Subject: [PATCH 17/17] remove fixEncoding for filenames --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 552cd17..4c5c1f6 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -6219,7 +6219,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No archiveinfo = {'buildroot_id': buildroot_id} archiveinfo['build_id'] = buildinfo['id'] if metadata_only: - filename = koji.fixEncoding(fileinfo['filename']) + filename = fileinfo['filename'] archiveinfo['filename'] = filename archiveinfo['size'] = fileinfo['filesize'] archiveinfo['checksum'] = fileinfo['checksum'] @@ -6230,7 +6230,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']] archiveinfo['metadata_only'] = True else: - filename = koji.fixEncoding(os.path.basename(filepath)) + filename = os.path.basename(filepath) archiveinfo['filename'] = filename archiveinfo['size'] = os.path.getsize(filepath) archivefp = open(filepath, 'rb') @@ -6349,7 +6349,7 @@ def _import_archive_file(filepath, destdir): be created. """ final_path = os.path.join(destdir, - koji.fixEncoding(os.path.basename(filepath))) + os.path.basename(filepath)) if os.path.exists(final_path): raise koji.GenericError("Error importing archive file, %s already exists" % final_path) if os.path.islink(filepath) or not os.path.isfile(filepath):