From 9b3af5f4924ab55dc7865124d8f2d2c7d271368b Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Feb 20 2023 09:30:25 +0000 Subject: [PATCH 1/2] Move db/auth to kojihub module Related: https://pagure.io/koji/issue/3666 --- diff --git a/docs/source/writing_koji_code.rst b/docs/source/writing_koji_code.rst index 07e750d..d668a74 100644 --- a/docs/source/writing_koji_code.rst +++ b/docs/source/writing_koji_code.rst @@ -675,7 +675,6 @@ Here are some guidelines on producing preferable pull requests. - ``cli/*`` - ``koji/__init__.py`` - - ``koji/auth.py`` - ``koji/tasks.py`` - ``koji/util.py`` - ``tests/test_lib/*`` diff --git a/koji/__init__.py b/koji/__init__.py index 756b9f6..024a8c5 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1834,7 +1834,7 @@ name=build def get_sequence_value(cursor, sequence): deprecated('Function get_sequence_value will be removed in Koji 1.34. ' - 'Use nextval function from koji.db.py.') + 'Use nextval function from kojihub.db.py.') cursor.execute("""SELECT nextval(%(sequence)s)""", locals()) return cursor.fetchone()[0] diff --git a/koji/auth.py b/koji/auth.py deleted file mode 100644 index d3406a3..0000000 --- a/koji/auth.py +++ /dev/null @@ -1,827 +0,0 @@ -# authentication module -# Copyright (c) 2005-2014 Red Hat, Inc. -# -# Koji is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; -# version 2.1 of the License. -# -# This software is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this software; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: -# Mike McLean -# Mike Bonnet - -from __future__ import absolute_import - -import logging -import random -import re -import socket -import string - -import six -from six.moves import range, urllib -import koji -from .context import context -from .util import to_list - -from koji.db import DeleteProcessor, InsertProcessor, QueryProcessor, UpdateProcessor, nextval - - -# 1 - load session if provided -# - check uri for session id -# - load session info from db -# - validate session -# 2 - create a session -# - maybe in two steps -# - - - -RetryWhitelist = [ - 'host.taskWait', - 'host.taskUnwait', - 'host.taskSetWait', - 'host.updateHost', - 'host.setBuildRootState', - 'repoExpire', - 'repoDelete', - 'repoProblem', -] - -AUTH_METHODS = ['login', 'sslLogin'] - -logger = logging.getLogger('koji.auth') - - -class Session(object): - - def __init__(self, args=None, hostip=None): - self.logged_in = False - self.id = None - self.master = None - self.key = None - self.user_id = None - self.authtype = None - self.hostip = None - self.user_data = {} - self.message = '' - self.exclusive = False - self.lockerror = None - self.callnum = None - # we look up perms, groups, and host_id on demand, see __getattr__ - self._perms = None - self._groups = None - self._host_id = '' - environ = getattr(context, 'environ', {}) - args = environ.get('QUERY_STRING', '') - # prefer new header-based sessions - if 'HTTP_KOJI_SESSION_ID' in environ: - self.id = int(environ['HTTP_KOJI_SESSION_ID']) - self.key = environ['HTTP_KOJI_SESSION_KEY'] - try: - callnum = int(environ['HTTP_KOJI_CALLNUM']) - except KeyError: - callnum = None - elif not context.opts['DisableURLSessions'] and args is not None: - # old deprecated method with session values in query string - # Option will be turned off by default in future release and removed later - if not args: - self.message = 'no session header or session args' - return - args = urllib.parse.parse_qs(args, strict_parsing=True) - try: - self.id = int(args['session-id'][0]) - self.key = args['session-key'][0] - except KeyError as field: - raise koji.AuthError('%s not specified in session args' % field) - try: - callnum = args['callnum'][0] - except Exception: - callnum = None - else: - self.message = 'no Koji-Session-* headers' - return - hostip = self.get_remote_ip(override=hostip) - # lookup the session - # sort for stability (unittests) - - fields = (('authtype', 'authtype'), ('callnum', 'callnum'), ('exclusive', 'exclusive'), - ('expired', 'expired'), ('master', 'master'), ('start_time', 'start_time'), - ('update_time', 'update_time'), ("date_part('epoch', start_time)", 'start_ts'), - ("date_part('epoch', update_time)", 'update_ts'), ('user_id', 'user_id')) - columns, aliases = zip(*fields) - - query = QueryProcessor(tables=['sessions'], columns=columns, aliases=aliases, - clauses=['id = %(id)i', 'key = %(key)s', 'hostip = %(hostip)s', - 'closed IS FALSE'], - values={'id': self.id, 'key': self.key, 'hostip': hostip}, - opts={'rowlock': True}) - session_data = query.executeOne(strict=False) - if not session_data: - query = QueryProcessor(tables=['sessions'], columns=['key', 'hostip'], - clauses=['id = %(id)i'], values={'id': self.id}) - row = query.executeOne(strict=False) - if row: - if self.key != row['key']: - logger.warning("Session ID %s is not related to session key %s.", - self.id, self.key) - elif hostip != row['hostip']: - logger.warning("Session ID %s is not related to host IP %s.", self.id, hostip) - raise koji.AuthError('Invalid session or bad credentials') - - # check for expiration - if session_data['expired']: - if getattr(context, 'method') not in AUTH_METHODS: - raise koji.AuthExpired('session "%s" has expired' % self.id) - - # check for callnum sanity - if callnum is not None: - try: - callnum = int(callnum) - except (ValueError, TypeError): - raise koji.AuthError("Invalid callnum: %r" % callnum) - lastcall = session_data['callnum'] - if lastcall is not None: - if lastcall > callnum: - raise koji.SequenceError("%s > %s (session %s)" % (lastcall, callnum, self.id)) - elif lastcall == callnum: - # Some explanation: - # This function is one of the few that performs its own commit. - # However, our storage of the current callnum is /after/ that - # commit. This means the the current callnum only gets committed if - # a commit happens afterward. - # We only schedule a commit for dml operations, so if we find the - # callnum in the db then a previous attempt succeeded but failed to - # return. Data was changed, so we cannot simply try the call again. - method = getattr(context, 'method', 'UNKNOWN') - if method not in RetryWhitelist: - raise koji.RetryError( - "unable to retry call %s (method %s) for session %s" % - (callnum, method, self.id)) - - if session_data['expired']: - return - - # read user data - # historical note: - # we used to get a row lock here as an attempt to maintain sanity of exclusive - # sessions, but it was an imperfect approach and the lock could cause some - # performance issues. - query = QueryProcessor(tables=['users'], columns=['name', 'status', 'usertype'], - clauses=['id=%(user_id)s'], - values={'user_id': session_data['user_id']}) - user_data = query.executeOne() - - if user_data['status'] != koji.USER_STATUS['NORMAL']: - raise koji.AuthError('logins by %s are not allowed' % user_data['name']) - # check for exclusive sessions - if session_data['exclusive']: - # we are the exclusive session for this user - self.exclusive = True - else: - # see if an exclusive session exists - query = QueryProcessor(tables=['sessions'], columns=['id'], - clauses=['user_id=%(user_id)s', 'exclusive = TRUE', - 'closed = FALSE'], - values=session_data) - excl_id = query.singleValue(strict=False) - - if excl_id: - if excl_id == session_data['master']: - # (note excl_id cannot be None) - # our master session has the lock - self.exclusive = True - else: - # a session unrelated to us has the lock - self.lockerror = "User locked by another session" - # we don't enforce here, but rely on the dispatcher to enforce - # if appropriate (otherwise it would be impossible to steal - # an exclusive session with the force option). - - # update timestamp - update = UpdateProcessor('sessions', rawdata={'update_time': 'NOW()'}, - clauses=['id = %(id)i'], values={'id': self.id}) - update.execute() - context.cnx.commit() - # update callnum (this is deliberately after the commit) - # see earlier note near RetryError - if callnum is not None: - update = UpdateProcessor('sessions', data={'callnum': callnum}, - clauses=['id = %(id)i'], values={'id': self.id}) - update.execute() - # we only want to commit the callnum change if there are other commits - context.commit_pending = False - - # record the login data - self.hostip = hostip - self.callnum = callnum - self.user_id = session_data['user_id'] - self.authtype = session_data['authtype'] - self.master = session_data['master'] - self.session_data = session_data - self.user_data = user_data - self.logged_in = True - - def __getattr__(self, name): - # grab perm and groups data on the fly - if name == 'perms': - if self._perms is None: - # in a dict for quicker lookup - self._perms = dict([[name, 1] for name in get_user_perms(self.user_id)]) - return self._perms - elif name == 'groups': - if self._groups is None: - self._groups = get_user_groups(self.user_id) - return self._groups - elif name == 'host_id': - if self._host_id == '': - self._host_id = self._getHostId() - return self._host_id - else: - raise AttributeError("%s" % name) - - def __str__(self): - # convenient display for debugging - if not self.logged_in: - s = "session: not logged in" - else: - s = "session %d: %r" % (self.id, self.__dict__) - if self.message: - s += " (%s)" % self.message - return s - - def validate(self): - if self.lockerror: - raise koji.AuthLockError(self.lockerror) - return True - - def get_remote_ip(self, override=None): - if not context.opts['CheckClientIP']: - return '-' - elif override is not None: - return override - else: - hostip = context.environ['REMOTE_ADDR'] - # XXX - REMOTE_ADDR not promised by wsgi spec - if hostip == '127.0.0.1': - hostip = socket.gethostbyname(socket.gethostname()) - return hostip - - def checkLoginAllowed(self, user_id): - """Verify that the user is allowed to login""" - query = QueryProcessor(tables=['users'], columns=['name', 'usertype', 'status'], - clauses=['id = %(user_id)i'], values={'user_id': user_id}) - result = query.executeOne(strict=False) - if not result: - raise koji.AuthError('invalid user_id: %s' % user_id) - - if result['status'] != koji.USER_STATUS['NORMAL']: - raise koji.AuthError('logins by %s are not allowed' % result['name']) - - def login(self, user, password, opts=None, renew=False, exclusive=False): - """create a login session""" - if opts is None: - opts = {} - if not isinstance(password, str) or len(password) == 0: - raise koji.AuthError('invalid username or password') - if self.logged_in: - raise koji.AuthError("Already logged in") - hostip = self.get_remote_ip(override=opts.get('hostip')) - - # check passwd - query = QueryProcessor(tables=['users'], columns=['id'], - clauses=['name = %(user)s', 'password = %(password)s'], - values={'user': user, 'password': password}) - user_id = query.singleValue(strict=False) - if not user_id: - raise koji.AuthError('invalid username or password') - - self.checkLoginAllowed(user_id) - - # create session and return - sinfo = self.createSession(user_id, hostip, koji.AUTHTYPES['NORMAL'], renew=renew) - if sinfo and exclusive and not self.exclusive: - self.makeExclusive() - context.cnx.commit() - return sinfo - - def getConnInfo(self): - """Return a tuple containing connection information - in the following format: - (local ip addr, local port, remote ip, remote port)""" - # For some reason req.connection.{local,remote}_addr contain port info, - # but no IP info. Use req.connection.{local,remote}_ip for that instead. - # See: http://lists.planet-lab.org/pipermail/devel-community/2005-June/001084.html - # local_ip seems to always be set to the same value as remote_ip, - # so get the local ip via a different method - local_ip = socket.gethostbyname(context.environ['SERVER_NAME']) - remote_ip = context.environ['REMOTE_ADDR'] - # XXX - REMOTE_ADDR not promised by wsgi spec - - # it appears that calling setports() with *any* value results in authentication - # failing with "Incorrect net address", so return 0 (which prevents - # python-krbV from calling setports()) - local_port = 0 - remote_port = 0 - - return (local_ip, local_port, remote_ip, remote_port) - - def sslLogin(self, proxyuser=None, proxyauthtype=None, renew=False, exclusive=None): - - """Login into brew via SSL. proxyuser name can be specified and if it is - allowed in the configuration file then connection is allowed to login as - that user. By default we assume that proxyuser is coming via same - authentication mechanism but proxyauthtype can be set to koji.AUTHTYPE['*'] - value for different handling. Typical case is proxying kerberos user via - web ui which itself is authenticated via SSL certificate. (See kojiweb - for usage). - - proxyauthtype is working only if AllowProxyAuthType option is set to - 'On' in the hub.conf - """ - if self.logged_in: - raise koji.AuthError("Already logged in") - - # we use REMOTE_USER to identify user - if context.environ.get('REMOTE_USER'): - # it is kerberos principal rather than user's name. - username = context.environ.get('REMOTE_USER') - client_dn = username - authtype = koji.AUTHTYPES['GSSAPI'] - else: - if context.environ.get('SSL_CLIENT_VERIFY') != 'SUCCESS': - raise koji.AuthError('could not verify client: %s' % - context.environ.get('SSL_CLIENT_VERIFY')) - - name_dn_component = context.opts.get('DNUsernameComponent', 'CN') - username = context.environ.get('SSL_CLIENT_S_DN_%s' % name_dn_component) - if not username: - raise koji.AuthError( - 'unable to get user information (%s) from client certificate' % - name_dn_component) - client_dn = context.environ.get('SSL_CLIENT_S_DN') - authtype = koji.AUTHTYPES['SSL'] - - if proxyuser: - if authtype == koji.AUTHTYPES['GSSAPI']: - delimiter = ',' - proxy_opt = 'ProxyPrincipals' - else: - delimiter = '|' - proxy_opt = 'ProxyDNs' - proxy_dns = [dn.strip() for dn in context.opts.get(proxy_opt, '').split(delimiter)] - - if client_dn in proxy_dns: - # the user authorized to login other users - username = proxyuser - else: - raise koji.AuthError('%s is not authorized to login other users' % client_dn) - - # in this point we can continue with proxied user in same way as if it is not proxied - if proxyauthtype is not None: - if not context.opts['AllowProxyAuthType'] and authtype != proxyauthtype: - raise koji.AuthError("Proxy must use same auth mechanism as hub (behaviour " - "can be overriden via AllowProxyAuthType hub option)") - if proxyauthtype not in (koji.AUTHTYPES['GSSAPI'], koji.AUTHTYPES['SSL']): - raise koji.AuthError( - "Proxied authtype %s is not valid for sslLogin" % proxyauthtype) - authtype = proxyauthtype - - if authtype == koji.AUTHTYPES['GSSAPI'] and '@' in username: - user_id = self.getUserIdFromKerberos(username) - else: - user_id = self.getUserId(username) - if not user_id: - if context.opts.get('LoginCreatesUser'): - if authtype == koji.AUTHTYPES['GSSAPI'] and '@' in username: - user_id = self.createUserFromKerberos(username) - else: - user_id = self.createUser(username) - else: - raise koji.AuthError('Unknown user: %s' % username) - - self.checkLoginAllowed(user_id) - - hostip = self.get_remote_ip() - - sinfo = self.createSession(user_id, hostip, authtype, renew=renew) - if sinfo and exclusive and not self.exclusive: - self.makeExclusive() - return sinfo - - def makeExclusive(self, force=False): - """Make this session exclusive""" - if self.master is not None: - raise koji.GenericError("subsessions cannot become exclusive") - if self.exclusive: - # shouldn't happen - raise koji.GenericError("session is already exclusive") - user_id = self.user_id - session_id = self.id - # acquire a row lock on the user entry - query = QueryProcessor(tables=['users'], columns=['id'], clauses=['id=%(user_id)s'], - values={'user_id': user_id}, opts={'rowlock': True}) - query.execute() - # check that no other sessions for this user are exclusive (including expired) - query = QueryProcessor(tables=['sessions'], columns=['id'], - clauses=['user_id=%(user_id)s', 'closed = FALSE', - 'exclusive = TRUE'], - values={'user_id': user_id}, opts={'rowlock': True}) - excl_id = query.singleValue(strict=False) - if excl_id: - if force: - # close the previous exclusive sessions and try again - update = UpdateProcessor('sessions', - data={'expired': True, 'exclusive': None, 'closed': True}, - clauses=['id=%(excl_id)s'], values={'excl_id': excl_id},) - update.execute() - else: - raise koji.AuthLockError("Cannot get exclusive session") - # mark this session exclusive - update = UpdateProcessor('sessions', data={'exclusive': True}, - clauses=['id=%(session_id)s'], values={'session_id': session_id}) - update.execute() - context.cnx.commit() - - def makeShared(self): - """Drop out of exclusive mode""" - session_id = self.id - update = UpdateProcessor('sessions', data={'exclusive': None}, - clauses=['id=%(session_id)s'], values={'session_id': session_id}) - update.execute() - context.cnx.commit() - - def logout(self, session_id=None): - """close a login session""" - if not self.logged_in: - # XXX raise an error? - raise koji.AuthError("Not logged in") - - if session_id: - if not context.session.hasPerm('admin'): - query = QueryProcessor(tables=['sessions'], columns=['id'], - clauses=['user_id = %(user_id)i', 'id = %(session_id)s'], - values={'user_id': self.user_id, 'session_id': session_id}) - if not query.singleValue(): - raise koji.ActionNotAllowed('only admins or owner may logout other session') - ses_id = session_id - else: - ses_id = self.id - update = UpdateProcessor('sessions', - data={'expired': True, 'exclusive': None, 'closed': True}, - clauses=['id = %(id)i OR master = %(id)i'], - values={'id': ses_id}) - update.execute() - context.cnx.commit() - if not session_id: - self.logged_in = False - - def logoutChild(self, session_id): - """close a subsession""" - if not self.logged_in: - # XXX raise an error? - raise koji.AuthError("Not logged in") - update = UpdateProcessor('sessions', - data={'expired': True, 'exclusive': None, 'closed': True}, - clauses=['id = %(session_id)i', 'master = %(master)i'], - values={'session_id': session_id, 'master': self.id}) - update.execute() - context.cnx.commit() - - def createSession(self, user_id, hostip, authtype, master=None, renew=False): - """Create a new session for the given user. - - Return a map containing the session-id and session-key. - If master is specified, create a subsession - """ - # generate a random key - alnum = string.ascii_letters + string.digits - key = "%s-%s" % (user_id, - ''.join([random.choice(alnum) for x in range(1, 20)])) - # use sha? sha.new(phrase).hexdigest() - - if renew and self.id is not None: - # just update key - session_id = self.id - self.key = key - if self.master: - # check if master session died meanwhile (expired is ok) - query = QueryProcessor(tables=['sessions'], - clauses=['id = %(master_id)d', 'closed IS FALSE'], - values={'master_id': self.master}, - opts={'countOnly': True}) - if query.executeOne() == 0: - return None - - update = UpdateProcessor('sessions', - clauses=['id=%(id)i'], - rawdata={'update_time': 'NOW()'}, - data={'key': self.key, 'expired': False}, - values={'id': self.id}) - update.execute() - else: - # get a session id - session_id = nextval('sessions_id_seq') - # add session id to database - insert = InsertProcessor('sessions', - data={'id': session_id, 'user_id': user_id, 'key': key, - 'hostip': hostip, 'authtype': authtype, - 'master': master}) - insert.execute() - context.cnx.commit() - - # return session info - return { - 'session-id': session_id, - 'session-key': key, - 'header-auth': True, # signalize to client to use new session handling in 1.30 - } - - def subsession(self): - "Create a subsession" - if not self.logged_in: - raise koji.AuthError("Not logged in") - master = self.master - if master is None: - master = self.id - return self.createSession(self.user_id, self.hostip, self.authtype, master=master) - - def getPerms(self): - if not self.logged_in: - return [] - return to_list(self.perms.keys()) - - def hasPerm(self, name): - if not self.logged_in: - return False - return name in self.perms - - def assertPerm(self, name): - if not self.hasPerm(name) and not self.hasPerm('admin'): - msg = "%s permission required" % name - if self.logged_in: - msg += ' (logged in as %s)' % self.user_data['name'] - else: - msg += ' (user not logged in)' - raise koji.ActionNotAllowed(msg) - - def assertLogin(self): - if not self.logged_in: - raise koji.ActionNotAllowed("you must be logged in for this operation") - - def hasGroup(self, group_id): - if not self.logged_in: - return False - # groups indexed by id - return group_id in self.groups - - def isUser(self, user_id): - if not self.logged_in: - return False - return (self.user_id == user_id or self.hasGroup(user_id)) - - def assertUser(self, user_id): - if not self.isUser(user_id) and not self.hasPerm('admin'): - raise koji.ActionNotAllowed("not owner") - - def _getHostId(self): - '''Using session data, find host id (if there is one)''' - if self.user_id is None: - return None - query = QueryProcessor(tables=['host'], columns=['id'], clauses=['user_id = %(uid)d'], - values={'uid': self.user_id}) - return query.singleValue(strict=False) - - def getHostId(self): - # for compatibility - return self.host_id - - def getUserId(self, username): - """Return the user ID associated with a particular username. If no user - with the given username if found, return None.""" - query = QueryProcessor(tables=['users'], columns=['id'], clauses=['name = %(username)s'], - values={'username': username}) - return query.singleValue(strict=False) - - def getUserIdFromKerberos(self, krb_principal): - """Return the user ID associated with a particular Kerberos principal. - If no user with the given princpal if found, return None.""" - self.checkKrbPrincipal(krb_principal) - query = QueryProcessor(tables=['users'], columns=['id'], - joins=['user_krb_principals ON ' - 'users.id = user_krb_principals.user_id'], - clauses=['krb_principal = %(krb_principal)s'], - values={'krb_principal': krb_principal}) - return query.singleValue(strict=False) - - def createUser(self, name, usertype=None, status=None, krb_principal=None, - krb_princ_check=True): - """ - Create a new user, using the provided values. - Return the user_id of the newly-created user. - """ - if not name: - raise koji.GenericError('a user must have a non-empty name') - - if usertype is None: - usertype = koji.USERTYPES['NORMAL'] - elif not koji.USERTYPES.get(usertype): - raise koji.GenericError('invalid user type: %s' % usertype) - - if status is None: - status = koji.USER_STATUS['NORMAL'] - elif not koji.USER_STATUS.get(status): - raise koji.GenericError('invalid status: %s' % status) - - # check if krb_principal is allowed - if krb_princ_check: - self.checkKrbPrincipal(krb_principal) - - user_id = nextval('users_id_seq') - - insert = InsertProcessor('users', - data={'id': user_id, 'name': name, 'usertype': usertype, - 'status': status}) - insert.execute() - if krb_principal: - insert = InsertProcessor('user_krb_principals', - data={'user_id': user_id, 'krb_principal': krb_principal}) - insert.execute() - context.cnx.commit() - - return user_id - - def setKrbPrincipal(self, name, krb_principal, krb_princ_check=True): - if krb_princ_check: - self.checkKrbPrincipal(krb_principal) - if isinstance(name, six.integer_types): - clauses = ['id = %(name)i'] - else: - clauses = ['name = %(name)s'] - query = QueryProcessor(tables=['users'], columns=['id'], clauses=clauses, - values={'name': name}) - user_id = query.singleValue(strict=False) - if not user_id: - context.cnx.rollback() - raise koji.AuthError('No such user: %s' % name) - insert = InsertProcessor('user_krb_principals', - data={'user_id': user_id, 'krb_principal': krb_principal}) - insert.execute() - context.cnx.commit() - return user_id - - def removeKrbPrincipal(self, name, krb_principal): - clauses = ['krb_principal = %(krb_principal)s'] - if isinstance(name, six.integer_types): - clauses.extend(['id = %(name)i']) - else: - clauses.extend(['name = %(name)s']) - query = QueryProcessor(tables=['users'], columns=['id'], - joins=['user_krb_principals ' - 'ON users.id = user_krb_principals.user_id'], - clauses=clauses, - values={'krb_principal': krb_principal, 'name': name}) - user_id = query.singleValue(strict=False) - if not user_id: - context.cnx.rollback() - raise koji.AuthError( - 'cannot remove Kerberos Principal:' - ' %(krb_principal)s with user %(name)s' % locals()) - cursor = context.cnx.cursor() - delete = DeleteProcessor(table='user_krb_principals', - clauses=['user_id = %(user_id)i', - 'krb_principal = %(krb_principal)s'], - values={'user_id': user_id, 'krb_principal': krb_principal}) - delete.execute() - context.cnx.commit() - return user_id - - def createUserFromKerberos(self, krb_principal): - """Create a new user, based on the Kerberos principal. Their - username will be everything before the "@" in the principal. - Return the ID of the newly created user.""" - atidx = krb_principal.find('@') - if atidx == -1: - raise koji.AuthError('invalid Kerberos principal: %s' % krb_principal) - user_name = krb_principal[:atidx] - - # check if user already exists - query = QueryProcessor(tables=['users'], columns=['id', 'krb_principal'], - joins=['LEFT JOIN user_krb_principals ON ' - 'users.id = user_krb_principals.user_id'], - clauses=['name = %(user_name)s'], - values={'user_name': user_name}) - r = query.execute() - if not r: - return self.createUser(user_name, krb_principal=krb_principal, - krb_princ_check=False) - else: - existing_user_krb_princs = [row['krb_principal'] for row in r] - if krb_principal in existing_user_krb_princs: - # do not set Kerberos principal if it already exists - return r[0]['id'] - return self.setKrbPrincipal(user_name, krb_principal, krb_princ_check=False) - - def checkKrbPrincipal(self, krb_principal): - """Check if the Kerberos principal is allowed""" - if krb_principal is None: - return - allowed_realms = context.opts.get('AllowedKrbRealms', '*') - if allowed_realms == '*': - return - allowed_realms = re.split(r'\s*,\s*', allowed_realms) - atidx = krb_principal.find('@') - if atidx == -1 or atidx == len(krb_principal) - 1: - raise koji.AuthError( - 'invalid Kerberos principal: %s' % krb_principal) - realm = krb_principal[atidx + 1:] - if realm not in allowed_realms: - raise koji.AuthError( - "Kerberos principal's realm: %s is not allowed" % realm) - - -def get_user_groups(user_id): - """Get user groups - - returns a dictionary where the keys are the group ids and the values - are the group names""" - t_group = koji.USERTYPES['GROUP'] - query = QueryProcessor(tables=['user_groups'], columns=['group_id', 'name'], - clauses=['active = TRUE', 'users.usertype=%(t_group)i', - 'user_id=%(user_id)i'], - joins=['users ON group_id = users.id'], - values={'t_group': t_group, 'user_id': user_id}) - return query.execute() - - -def get_user_perms(user_id): - query = QueryProcessor(tables=['user_perms'], columns=['name'], - clauses=['active = TRUE', 'user_id=%(user_id)s'], - joins=['permissions ON perm_id = permissions.id'], - values={'user_id': user_id}) - result = query.execute() - return [r['name'] for r in result] - - -def get_user_data(user_id): - query = QueryProcessor(tables=['users'], columns=['name', 'status', 'usertype'], - clauses=['id=%(user_id)s'], values={'user_id': user_id}) - return query.executeOne(strict=False) - - -def login(*args, **opts): - """Create a login session with plain user/password credentials. - - :param str user: username - :param str password: password - :param dict opts: curently can contain only 'host_ip' key for overriding client IP address - - :returns dict: session info - """ - - return context.session.login(*args, **opts) - - -def sslLogin(*args, **opts): - """Login via SSL certificate - - :param str proxyuser: proxy username - :returns dict: session info - """ - return context.session.sslLogin(*args, **opts) - - -def logout(session_id=None): - """expire a login session""" - return context.session.logout(session_id) - - -def subsession(): - """Create a subsession""" - return context.session.subsession() - - -def logoutChild(session_id): - """expire a subsession - - :param int subsession_id: subsession ID (for current session) - """ - return context.session.logoutChild(session_id) - - -def exclusiveSession(*args, **opts): - """Make this session exclusive""" - return context.session.makeExclusive(*args, **opts) - - -def sharedSession(): - """Drop out of exclusive mode""" - return context.session.makeShared() diff --git a/koji/db.py b/koji/db.py deleted file mode 100644 index 478c30f..0000000 --- a/koji/db.py +++ /dev/null @@ -1,924 +0,0 @@ -# python library - -# db utilities for koji -# Copyright (c) 2005-2014 Red Hat, Inc. -# -# Koji is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; -# version 2.1 of the License. -# -# This software is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this software; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: -# Mike McLean - - -from __future__ import absolute_import - -import logging -import koji -import os -# import psycopg2.extensions -# # don't convert timestamp fields to DateTime objects -# del psycopg2.extensions.string_types[1114] -# del psycopg2.extensions.string_types[1184] -# del psycopg2.extensions.string_types[1082] -# del psycopg2.extensions.string_types[1083] -# del psycopg2.extensions.string_types[1266] -import re -import sys -import time -import traceback - -import psycopg2 - -import koji.context -context = koji.context.context - - -POSITIONAL_RE = re.compile(r'%[a-z]') -NAMED_RE = re.compile(r'%\(([^\)]+)\)[a-z]') - -## Globals ## -_DBopts = None -# A persistent connection to the database. -# A new connection will be created whenever -# Apache forks a new worker, and that connection -# will be used to service all requests handled -# by that worker. -# This probably doesn't need to be a ThreadLocal -# since Apache is not using threading, -# but play it safe anyway. -_DBconn = koji.context.ThreadLocal() - -logger = logging.getLogger('koji.db') - - -class DBWrapper: - def __init__(self, cnx): - self.cnx = cnx - - def __getattr__(self, key): - if not self.cnx: - raise Exception('connection is closed') - return getattr(self.cnx, key) - - def cursor(self, *args, **kw): - if not self.cnx: - raise Exception('connection is closed') - return CursorWrapper(self.cnx.cursor(*args, **kw)) - - def close(self): - # Rollback any uncommitted changes and clear the connection so - # this DBWrapper is no longer usable after close() - if not self.cnx: - raise Exception('connection is closed') - self.cnx.cursor().execute('ROLLBACK') - # We do this rather than cnx.rollback to avoid opening a new transaction - # If our connection gets recycled cnx.rollback will be called then. - self.cnx = None - - -class CursorWrapper: - def __init__(self, cursor): - self.cursor = cursor - self.logger = logging.getLogger('koji.db') - - def __getattr__(self, key): - return getattr(self.cursor, key) - - def _timed_call(self, method, args, kwargs): - start = time.time() - ret = getattr(self.cursor, method)(*args, **kwargs) - self.logger.debug("%s operation completed in %.4f seconds", method, time.time() - start) - return ret - - def fetchone(self, *args, **kwargs): - return self._timed_call('fetchone', args, kwargs) - - def fetchall(self, *args, **kwargs): - return self._timed_call('fetchall', args, kwargs) - - def quote(self, operation, parameters): - if hasattr(self.cursor, "mogrify"): - quote = self.cursor.mogrify - else: - def quote(a, b): - return a % b - try: - return quote(operation, parameters) - except Exception: - self.logger.exception( - 'Unable to quote query:\n%s\nParameters: %s', operation, parameters) - return "INVALID QUERY" - - def preformat(self, sql, params): - """psycopg2 requires all variable placeholders to use the string (%s) datatype, - regardless of the actual type of the data. Format the sql string to be compliant. - It also requires IN parameters to be in tuple rather than list format.""" - sql = POSITIONAL_RE.sub(r'%s', sql) - sql = NAMED_RE.sub(r'%(\1)s', sql) - if isinstance(params, dict): - for name, value in params.items(): - if isinstance(value, list): - params[name] = tuple(value) - else: - if isinstance(params, tuple): - params = list(params) - for i, item in enumerate(params): - if isinstance(item, list): - params[i] = tuple(item) - return sql, params - - def execute(self, operation, parameters=(), log_errors=True): - debug = self.logger.isEnabledFor(logging.DEBUG) - operation, parameters = self.preformat(operation, parameters) - if debug: - self.logger.debug(self.quote(operation, parameters)) - start = time.time() - try: - ret = self.cursor.execute(operation, parameters) - except Exception: - if log_errors: - self.logger.error('Query failed. Query was: %s', self.quote(operation, parameters)) - raise - if debug: - self.logger.debug("Execute operation completed in %.4f seconds", time.time() - start) - return ret - - -## Functions ## -def provideDBopts(**opts): - global _DBopts - if _DBopts is None: - _DBopts = dict([i for i in opts.items() if i[1] is not None]) - - -def setDBopts(**opts): - global _DBopts - _DBopts = opts - - -def getDBopts(): - return _DBopts - - -def connect(): - logger = logging.getLogger('koji.db') - global _DBconn - if hasattr(_DBconn, 'conn'): - # Make sure the previous transaction has been - # closed. This is safe to call multiple times. - conn = _DBconn.conn - try: - # Under normal circumstances, the last use of this connection - # will have issued a raw ROLLBACK to close the transaction. To - # avoid 'no transaction in progress' warnings (depending on postgres - # configuration) we open a new one here. - # Should there somehow be a transaction in progress, a second - # BEGIN will be a harmless no-op, though there may be a warning. - conn.cursor().execute('BEGIN') - conn.rollback() - return DBWrapper(conn) - except psycopg2.Error: - del _DBconn.conn - # create a fresh connection - opts = _DBopts - if opts is None: - opts = {} - try: - if 'dsn' in opts: - conn = psycopg2.connect(dsn=opts['dsn']) - else: - conn = psycopg2.connect(**opts) - conn.set_client_encoding('UTF8') - except Exception: - logger.error(''.join(traceback.format_exception(*sys.exc_info()))) - raise - # XXX test - # return conn - _DBconn.conn = conn - - return DBWrapper(conn) - - -def _dml(operation, values, log_errors=True): - """Run an insert, update, or delete. Return number of rows affected - If log is False, errors will not be logged. It makes sense only for - queries which are expected to fail (LOCK NOWAIT) - """ - c = context.cnx.cursor() - c.execute(operation, values, log_errors=log_errors) - ret = c.rowcount - logger.debug("Operation affected %s row(s)", ret) - c.close() - context.commit_pending = True - return ret - - -def _fetchMulti(query, values): - """Run the query and return all rows""" - c = context.cnx.cursor() - c.execute(query, values) - results = c.fetchall() - c.close() - return results - - -def _fetchSingle(query, values, strict=False): - """Run the query and return a single row - - If strict is true, raise an error if the query returns more or less than - one row.""" - results = _fetchMulti(query, values) - numRows = len(results) - if numRows == 0: - if strict: - raise koji.GenericError('query returned no rows') - else: - return None - elif strict and numRows > 1: - raise koji.GenericError('multiple rows returned for a single row query') - else: - return results[0] - - -def _singleValue(query, values=None, strict=True): - """Perform a query that returns a single value. - - Note that unless strict is True a return value of None could mean either - a single NULL value or zero rows returned.""" - if values is None: - values = {} - row = _fetchSingle(query, values, strict) - if row: - if strict and len(row) > 1: - raise koji.GenericError('multiple fields returned for a single value query') - return row[0] - else: - # don't need to check strict here, since that was already handled by _singleRow() - return None - - -def _multiRow(query, values, fields): - """Return all rows from "query". Named query parameters - can be specified using the "values" map. Results will be returned - as a list of maps. Each map in the list will have a key for each - element in the "fields" list. If there are no results, an empty - list will be returned.""" - return [dict(zip(fields, row)) for row in _fetchMulti(query, values)] - - -def _singleRow(query, values, fields, strict=False): - """Return a single row from "query". Named parameters can be - specified using the "values" map. The result will be returned as - as map. The map will have a key for each element in the "fields" - list. If more than one row is returned and "strict" is true, a - GenericError will be raised. If no rows are returned, and "strict" - is True, a GenericError will be raised. Otherwise None will be - returned.""" - row = _fetchSingle(query, values, strict) - if row: - return dict(zip(fields, row)) - else: - # strict enforced by _fetchSingle - return None - - -def get_event(): - """Get an event id for this transaction - - We cache the result in context, so subsequent calls in the same transaction will - get the same event. - - This cache is cleared between the individual calls in a multicall. - See: https://pagure.io/koji/pull-request/74 - """ - if hasattr(context, 'event_id'): - return context.event_id - event_id = _singleValue("SELECT get_event()") - context.event_id = event_id - return event_id - - -def nextval(sequence): - """Get the next value for the given sequence""" - data = {'sequence': sequence} - return _singleValue("SELECT nextval(%(sequence)s)", data, strict=True) - - -def currval(sequence): - """Get the current value for the given sequence""" - data = {'sequence': sequence} - return _singleValue("SELECT currval(%(sequence)s)", data, strict=True) - - -class Savepoint(object): - - def __init__(self, name): - self.name = name - _dml("SAVEPOINT %s" % name, {}) - - def rollback(self): - _dml("ROLLBACK TO SAVEPOINT %s" % self.name, {}) - - -class InsertProcessor(object): - """Build an insert statement - - table - the table to insert into - data - a dictionary of data to insert (keys = row names) - rawdata - data to insert specified as sql expressions rather than python values - - does not support query inserts of "DEFAULT VALUES" - """ - - def __init__(self, table, data=None, rawdata=None): - self.table = table - self.data = {} - if data: - self.data.update(data) - self.rawdata = {} - if rawdata: - self.rawdata.update(rawdata) - - def __str__(self): - if not self.data and not self.rawdata: - return "-- incomplete update: no assigns" - parts = ['INSERT INTO %s ' % self.table] - columns = sorted(list(self.data.keys()) + list(self.rawdata.keys())) - parts.append("(%s) " % ', '.join(columns)) - values = [] - for key in columns: - if key in self.data: - values.append("%%(%s)s" % key) - else: - values.append("(%s)" % self.rawdata[key]) - parts.append("VALUES (%s)" % ', '.join(values)) - return ''.join(parts) - - def __repr__(self): - return "" % vars(self) - - def set(self, **kwargs): - """Set data via keyword args""" - self.data.update(kwargs) - - def rawset(self, **kwargs): - """Set rawdata via keyword args""" - self.rawdata.update(kwargs) - - def make_create(self, event_id=None, user_id=None): - if event_id is None: - event_id = get_event() - if user_id is None: - context.session.assertLogin() - user_id = context.session.user_id - self.data['create_event'] = event_id - self.data['creator_id'] = user_id - - def dup_check(self): - """Check to see if the insert duplicates an existing row""" - if self.rawdata: - logger.warning("Can't perform duplicate check") - return None - data = self.data.copy() - if 'create_event' in self.data: - # versioned table - data['active'] = True - del data['create_event'] - del data['creator_id'] - clauses = ["%s = %%(%s)s" % (k, k) for k in data] - query = QueryProcessor(columns=list(data.keys()), tables=[self.table], - clauses=clauses, values=data) - if query.execute(): - return True - return False - - def execute(self): - return _dml(str(self), self.data) - - -class UpdateProcessor(object): - """Build an update statement - - table - the table to insert into - data - a dictionary of data to insert (keys = row names) - rawdata - data to insert specified as sql expressions rather than python values - clauses - a list of where clauses which will be ANDed together - values - dict of values used in clauses - - does not support the FROM clause - """ - - def __init__(self, table, data=None, rawdata=None, clauses=None, values=None): - self.table = table - self.data = {} - if data: - self.data.update(data) - self.rawdata = {} - if rawdata: - self.rawdata.update(rawdata) - self.clauses = [] - if clauses: - self.clauses.extend(clauses) - self.values = {} - if values: - self.values.update(values) - - def __str__(self): - if not self.data and not self.rawdata: - return "-- incomplete update: no assigns" - parts = ['UPDATE %s SET ' % self.table] - assigns = ["%s = %%(data.%s)s" % (key, key) for key in self.data] - assigns.extend(["%s = (%s)" % (key, self.rawdata[key]) for key in self.rawdata]) - parts.append(', '.join(sorted(assigns))) - if self.clauses: - parts.append('\nWHERE ') - parts.append(' AND '.join(["( %s )" % c for c in sorted(self.clauses)])) - return ''.join(parts) - - def __repr__(self): - return "" % vars(self) - - def get_values(self): - """Returns unified values dict, including data""" - ret = {} - ret.update(self.values) - for key in self.data: - ret["data." + key] = self.data[key] - return ret - - def set(self, **kwargs): - """Set data via keyword args""" - self.data.update(kwargs) - - def rawset(self, **kwargs): - """Set rawdata via keyword args""" - self.rawdata.update(kwargs) - - def make_revoke(self, event_id=None, user_id=None): - """Add standard revoke options to the update""" - if event_id is None: - event_id = get_event() - if user_id is None: - context.session.assertLogin() - user_id = context.session.user_id - self.data['revoke_event'] = event_id - self.data['revoker_id'] = user_id - self.rawdata['active'] = 'NULL' - self.clauses.append('active = TRUE') - - def execute(self): - return _dml(str(self), self.get_values()) - - -class DeleteProcessor(object): - """Build an delete statement - - table - the table to delete - clauses - a list of where clauses which will be ANDed together - values - dict of values used in clauses - """ - - def __init__(self, table, clauses=None, values=None): - self.table = table - self.clauses = [] - if clauses: - self.clauses.extend(clauses) - self.values = {} - if values: - self.values.update(values) - - def __str__(self): - parts = ['DELETE FROM %s ' % self.table] - if self.clauses: - parts.append('\nWHERE ') - parts.append(' AND '.join(["( %s )" % c for c in sorted(self.clauses)])) - return ''.join(parts) - - def __repr__(self): - return "" % vars(self) - - def get_values(self): - """Returns unified values dict, including data""" - ret = {} - ret.update(self.values) - return ret - - def execute(self): - return _dml(str(self), self.get_values()) - - -class QueryProcessor(object): - """ - Build a query from its components. - - columns, aliases, tables: lists of the column names to retrieve, - the tables to retrieve them from, and the key names to use when - returning values as a map, respectively - - joins: a list of joins in the form 'table1 ON table1.col1 = table2.col2', 'JOIN' will be - prepended automatically; if extended join syntax (LEFT, OUTER, etc.) is required, - it can be specified, and 'JOIN' will not be prepended - - clauses: a list of where clauses in the form 'table1.col1 OPER table2.col2-or-variable'; - each clause will be surrounded by parentheses and all will be AND'ed together - - values: the map that will be used to replace any substitution expressions in the query - - transform: a function that will be called on each row (not compatible with - countOnly or singleValue) - - opts: a map of query options; currently supported options are: - countOnly: if True, return an integer indicating how many results would have been - returned, rather than the actual query results - order: a column or alias name to use in the 'ORDER BY' clause - offset: an integer to use in the 'OFFSET' clause - limit: an integer to use in the 'LIMIT' clause - asList: if True, return results as a list of lists, where each list contains the - column values in query order, rather than the usual list of maps - rowlock: if True, use "FOR UPDATE" to lock the queried rows - group: a column or alias name to use in the 'GROUP BY' clause - (controlled by enable_group) - - enable_group: if True, opts.group will be enabled - """ - - iterchunksize = 1000 - - def __init__(self, columns=None, aliases=None, tables=None, - joins=None, clauses=None, values=None, transform=None, - opts=None, enable_group=False): - self.columns = columns - self.aliases = aliases - if columns and aliases: - if len(columns) != len(aliases): - raise Exception('column and alias lists must be the same length') - # reorder - alias_table = sorted(zip(aliases, columns)) - self.aliases = [x[0] for x in alias_table] - self.columns = [x[1] for x in alias_table] - self.colsByAlias = dict(alias_table) - else: - self.colsByAlias = {} - if columns: - self.columns = sorted(columns) - if aliases: - self.aliases = sorted(aliases) - self.tables = tables - self.joins = joins - if clauses: - self.clauses = sorted(clauses) - else: - self.clauses = clauses - self.cursors = 0 - if values: - self.values = values - else: - self.values = {} - self.transform = transform - if opts: - self.opts = opts - else: - self.opts = {} - self.enable_group = enable_group - self.logger = logging.getLogger('koji.db') - - def countOnly(self, count): - self.opts['countOnly'] = count - - def __str__(self): - query = \ - """ -SELECT %(col_str)s - FROM %(table_str)s -%(join_str)s -%(clause_str)s - %(group_str)s - %(order_str)s -%(offset_str)s - %(limit_str)s -""" - if self.opts.get('countOnly'): - if self.opts.get('offset') \ - or self.opts.get('limit') \ - or (self.enable_group and self.opts.get('group')): - # If we're counting with an offset and/or limit, we need - # to wrap the offset/limited query and then count the results, - # rather than trying to offset/limit the single row returned - # by count(*). Because we're wrapping the query, we don't care - # about the column values. - col_str = '1' - else: - col_str = 'count(*)' - else: - col_str = self._seqtostr(self.columns) - table_str = self._seqtostr(self.tables, sort=True) - join_str = self._joinstr() - clause_str = self._seqtostr(self.clauses, sep=')\n AND (') - if clause_str: - clause_str = ' WHERE (' + clause_str + ')' - if self.enable_group: - group_str = self._group() - else: - group_str = '' - order_str = self._order() - offset_str = self._optstr('offset') - limit_str = self._optstr('limit') - - query = query % locals() - if self.opts.get('countOnly') and \ - (self.opts.get('offset') or - self.opts.get('limit') or - (self.enable_group and self.opts.get('group'))): - query = 'SELECT count(*)\nFROM (' + query + ') numrows' - if self.opts.get('rowlock'): - query += '\n FOR UPDATE' - return query - - def __repr__(self): - return '' % \ - (self.columns, self.aliases, self.tables, self.joins, self.clauses, self.values, - self.opts) - - def _seqtostr(self, seq, sep=', ', sort=False): - if seq: - if sort: - seq = sorted(seq) - return sep.join(seq) - else: - return '' - - def _joinstr(self): - if not self.joins: - return '' - result = '' - for join in self.joins: - if result: - result += '\n' - if re.search(r'\bjoin\b', join, re.IGNORECASE): - # The join clause already contains the word 'join', - # so don't prepend 'JOIN' to it - result += ' ' + join - else: - result += ' JOIN ' + join - return result - - def _order(self): - # Don't bother sorting if we're just counting - if self.opts.get('countOnly'): - return '' - order_opt = self.opts.get('order') - if order_opt: - order_exprs = [] - for order in order_opt.split(','): - if order.startswith('-'): - order = order[1:] - direction = ' DESC' - else: - direction = '' - # Check if we're ordering by alias first - orderCol = self.colsByAlias.get(order) - if orderCol: - pass - elif order in self.columns: - orderCol = order - else: - raise Exception('Invalid order: ' + order) - order_exprs.append(orderCol + direction) - return 'ORDER BY ' + ', '.join(order_exprs) - else: - return '' - - def _group(self): - group_opt = self.opts.get('group') - if group_opt: - group_exprs = [] - for group in group_opt.split(','): - if group: - group_exprs.append(group) - return 'GROUP BY ' + ', '.join(group_exprs) - else: - return '' - - def _optstr(self, optname): - optval = self.opts.get(optname) - if optval: - return '%s %i' % (optname.upper(), optval) - else: - return '' - - def singleValue(self, strict=True): - # self.transform not applied here - return _singleValue(str(self), self.values, strict=strict) - - def execute(self): - query = str(self) - if self.opts.get('countOnly'): - return _singleValue(query, self.values, strict=True) - elif self.opts.get('asList'): - if self.transform is None: - return _fetchMulti(query, self.values) - else: - # if we're transforming, generate the dicts so the transform can modify - fields = self.aliases or self.columns - data = _multiRow(query, self.values, fields) - data = [self.transform(row) for row in data] - # and then convert back to lists - data = [[row[f] for f in fields] for row in data] - return data - else: - data = _multiRow(query, self.values, (self.aliases or self.columns)) - if self.transform is not None: - data = [self.transform(row) for row in data] - return data - - def iterate(self): - if self.opts.get('countOnly'): - return self.execute() - elif self.opts.get('limit') and self.opts['limit'] < self.iterchunksize: - return self.execute() - else: - fields = self.aliases or self.columns - fields = list(fields) - cname = "qp_cursor_%s_%i_%i" % (id(self), os.getpid(), self.cursors) - self.cursors += 1 - self.logger.debug('Setting up query iterator. cname=%r', cname) - return self._iterate(cname, str(self), self.values.copy(), fields, - self.iterchunksize, self.opts.get('asList')) - - def _iterate(self, cname, query, values, fields, chunksize, as_list=False): - # We pass all this data into the generator so that the iterator works - # from the snapshot when it was generated. Otherwise reuse of the processor - # for similar queries could have unpredictable results. - query = "DECLARE %s NO SCROLL CURSOR FOR %s" % (cname, query) - c = context.cnx.cursor() - c.execute(query, values) - c.close() - try: - query = "FETCH %i FROM %s" % (chunksize, cname) - while True: - if as_list: - if self.transform is None: - buf = _fetchMulti(query, {}) - else: - # if we're transforming, generate the dicts so the transform can modify - buf = _multiRow(query, self.values, fields) - buf = [self.transform(row) for row in buf] - # and then convert back to lists - buf = [[row[f] for f in fields] for row in buf] - else: - buf = _multiRow(query, {}, fields) - if self.transform is not None: - buf = [self.transform(row) for row in buf] - if not buf: - break - for row in buf: - yield row - finally: - c = context.cnx.cursor() - c.execute("CLOSE %s" % cname) - c.close() - - def executeOne(self, strict=False): - results = self.execute() - if isinstance(results, list): - if len(results) > 0: - if strict and len(results) > 1: - raise koji.GenericError('multiple rows returned for a single row query') - return results[0] - elif strict: - raise koji.GenericError('query returned no rows') - else: - return None - return results - - -class BulkInsertProcessor(object): - def __init__(self, table, data=None, columns=None, strict=True, batch=1000): - """Do bulk inserts - it has some limitations compared to - InsertProcessor (no rawset, dup_check). - - set() is replaced with add_record() to avoid confusion - - table - name of the table - data - list of dict per record - columns - list/set of names of used columns - makes sense - mainly with strict=True - strict - if True, all records must contain values for all columns. - if False, missing values will be inserted as NULLs - batch - batch size for inserts (one statement per batch) - """ - - self.table = table - self.data = [] - if columns is None: - self.columns = set() - else: - self.columns = set(columns) - if data is not None: - self.data = data - for row in data: - self.columns |= set(row.keys()) - self.strict = strict - self.batch = batch - - def __str__(self): - if not self.data: - return "-- incomplete insert: no data" - query, params = self._get_insert(self.data) - return query - - def _get_insert(self, data): - """ - Generate one insert statement for the given data - - :param list data: list of rows (dict format) to insert - :returns: (query, params) - """ - - if not data: - # should not happen - raise ValueError('no data for insert') - parts = ['INSERT INTO %s ' % self.table] - columns = sorted(self.columns) - parts.append("(%s) " % ', '.join(columns)) - - prepared_data = {} - values = [] - i = 0 - for row in data: - row_values = [] - for key in columns: - if key in row: - row_key = '%s%d' % (key, i) - row_values.append("%%(%s)s" % row_key) - prepared_data[row_key] = row[key] - elif self.strict: - raise koji.GenericError("Missing value %s in BulkInsert" % key) - else: - row_values.append("NULL") - values.append("(%s)" % ', '.join(row_values)) - i += 1 - parts.append("VALUES %s" % ', '.join(values)) - return ''.join(parts), prepared_data - - def __repr__(self): - return "" % vars(self) - - def add_record(self, **kwargs): - """Set whole record via keyword args""" - if not kwargs: - raise koji.GenericError("Missing values in BulkInsert.add_record") - self.data.append(kwargs) - self.columns |= set(kwargs.keys()) - - def execute(self): - if not self.batch: - self._one_insert(self.data) - else: - for i in range(0, len(self.data), self.batch): - data = self.data[i:i + self.batch] - self._one_insert(data) - - def _one_insert(self, data): - query, params = self._get_insert(data) - _dml(query, params) - - -def _applyQueryOpts(results, queryOpts): - """ - Apply queryOpts to results in the same way QueryProcessor would. - results is a list of maps. - queryOpts is a map which may contain the following fields: - countOnly - order - offset - limit - - Note: - - asList is supported by QueryProcessor but not by this method. - We don't know the original query order, and so don't have a way to - return a useful list. asList should be handled by the caller. - - group is supported by QueryProcessor but not by this method as well. - """ - if queryOpts is None: - queryOpts = {} - if queryOpts.get('order'): - order = queryOpts['order'] - reverse = False - if order.startswith('-'): - order = order[1:] - reverse = True - results.sort(key=lambda o: o[order], reverse=reverse) - if queryOpts.get('offset'): - results = results[queryOpts['offset']:] - if queryOpts.get('limit'): - results = results[:queryOpts['limit']] - if queryOpts.get('countOnly'): - return len(results) - else: - return results diff --git a/kojihub/auth.py b/kojihub/auth.py new file mode 100644 index 0000000..9ddb7da --- /dev/null +++ b/kojihub/auth.py @@ -0,0 +1,827 @@ +# authentication module +# Copyright (c) 2005-2014 Red Hat, Inc. +# +# Koji is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; +# version 2.1 of the License. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this software; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: +# Mike McLean +# Mike Bonnet + +from __future__ import absolute_import + +import logging +import random +import re +import socket +import string + +import six +from six.moves import range, urllib +import koji +from koji.context import context +from koji.util import to_list + +from kojihub.db import DeleteProcessor, InsertProcessor, QueryProcessor, UpdateProcessor, nextval + + +# 1 - load session if provided +# - check uri for session id +# - load session info from db +# - validate session +# 2 - create a session +# - maybe in two steps +# - + + +RetryWhitelist = [ + 'host.taskWait', + 'host.taskUnwait', + 'host.taskSetWait', + 'host.updateHost', + 'host.setBuildRootState', + 'repoExpire', + 'repoDelete', + 'repoProblem', +] + +AUTH_METHODS = ['login', 'sslLogin'] + +logger = logging.getLogger('koji.auth') + + +class Session(object): + + def __init__(self, args=None, hostip=None): + self.logged_in = False + self.id = None + self.master = None + self.key = None + self.user_id = None + self.authtype = None + self.hostip = None + self.user_data = {} + self.message = '' + self.exclusive = False + self.lockerror = None + self.callnum = None + # we look up perms, groups, and host_id on demand, see __getattr__ + self._perms = None + self._groups = None + self._host_id = '' + environ = getattr(context, 'environ', {}) + args = environ.get('QUERY_STRING', '') + # prefer new header-based sessions + if 'HTTP_KOJI_SESSION_ID' in environ: + self.id = int(environ['HTTP_KOJI_SESSION_ID']) + self.key = environ['HTTP_KOJI_SESSION_KEY'] + try: + callnum = int(environ['HTTP_KOJI_CALLNUM']) + except KeyError: + callnum = None + elif not context.opts['DisableURLSessions'] and args is not None: + # old deprecated method with session values in query string + # Option will be turned off by default in future release and removed later + if not args: + self.message = 'no session header or session args' + return + args = urllib.parse.parse_qs(args, strict_parsing=True) + try: + self.id = int(args['session-id'][0]) + self.key = args['session-key'][0] + except KeyError as field: + raise koji.AuthError('%s not specified in session args' % field) + try: + callnum = args['callnum'][0] + except Exception: + callnum = None + else: + self.message = 'no Koji-Session-* headers' + return + hostip = self.get_remote_ip(override=hostip) + # lookup the session + # sort for stability (unittests) + + fields = (('authtype', 'authtype'), ('callnum', 'callnum'), ('exclusive', 'exclusive'), + ('expired', 'expired'), ('master', 'master'), ('start_time', 'start_time'), + ('update_time', 'update_time'), ("date_part('epoch', start_time)", 'start_ts'), + ("date_part('epoch', update_time)", 'update_ts'), ('user_id', 'user_id')) + columns, aliases = zip(*fields) + + query = QueryProcessor(tables=['sessions'], columns=columns, aliases=aliases, + clauses=['id = %(id)i', 'key = %(key)s', 'hostip = %(hostip)s', + 'closed IS FALSE'], + values={'id': self.id, 'key': self.key, 'hostip': hostip}, + opts={'rowlock': True}) + session_data = query.executeOne(strict=False) + if not session_data: + query = QueryProcessor(tables=['sessions'], columns=['key', 'hostip'], + clauses=['id = %(id)i'], values={'id': self.id}) + row = query.executeOne(strict=False) + if row: + if self.key != row['key']: + logger.warning("Session ID %s is not related to session key %s.", + self.id, self.key) + elif hostip != row['hostip']: + logger.warning("Session ID %s is not related to host IP %s.", self.id, hostip) + raise koji.AuthError('Invalid session or bad credentials') + + # check for expiration + if session_data['expired']: + if getattr(context, 'method') not in AUTH_METHODS: + raise koji.AuthExpired('session "%s" has expired' % self.id) + + # check for callnum sanity + if callnum is not None: + try: + callnum = int(callnum) + except (ValueError, TypeError): + raise koji.AuthError("Invalid callnum: %r" % callnum) + lastcall = session_data['callnum'] + if lastcall is not None: + if lastcall > callnum: + raise koji.SequenceError("%s > %s (session %s)" % (lastcall, callnum, self.id)) + elif lastcall == callnum: + # Some explanation: + # This function is one of the few that performs its own commit. + # However, our storage of the current callnum is /after/ that + # commit. This means the the current callnum only gets committed if + # a commit happens afterward. + # We only schedule a commit for dml operations, so if we find the + # callnum in the db then a previous attempt succeeded but failed to + # return. Data was changed, so we cannot simply try the call again. + method = getattr(context, 'method', 'UNKNOWN') + if method not in RetryWhitelist: + raise koji.RetryError( + "unable to retry call %s (method %s) for session %s" % + (callnum, method, self.id)) + + if session_data['expired']: + return + + # read user data + # historical note: + # we used to get a row lock here as an attempt to maintain sanity of exclusive + # sessions, but it was an imperfect approach and the lock could cause some + # performance issues. + query = QueryProcessor(tables=['users'], columns=['name', 'status', 'usertype'], + clauses=['id=%(user_id)s'], + values={'user_id': session_data['user_id']}) + user_data = query.executeOne() + + if user_data['status'] != koji.USER_STATUS['NORMAL']: + raise koji.AuthError('logins by %s are not allowed' % user_data['name']) + # check for exclusive sessions + if session_data['exclusive']: + # we are the exclusive session for this user + self.exclusive = True + else: + # see if an exclusive session exists + query = QueryProcessor(tables=['sessions'], columns=['id'], + clauses=['user_id=%(user_id)s', 'exclusive = TRUE', + 'closed = FALSE'], + values=session_data) + excl_id = query.singleValue(strict=False) + + if excl_id: + if excl_id == session_data['master']: + # (note excl_id cannot be None) + # our master session has the lock + self.exclusive = True + else: + # a session unrelated to us has the lock + self.lockerror = "User locked by another session" + # we don't enforce here, but rely on the dispatcher to enforce + # if appropriate (otherwise it would be impossible to steal + # an exclusive session with the force option). + + # update timestamp + update = UpdateProcessor('sessions', rawdata={'update_time': 'NOW()'}, + clauses=['id = %(id)i'], values={'id': self.id}) + update.execute() + context.cnx.commit() + # update callnum (this is deliberately after the commit) + # see earlier note near RetryError + if callnum is not None: + update = UpdateProcessor('sessions', data={'callnum': callnum}, + clauses=['id = %(id)i'], values={'id': self.id}) + update.execute() + # we only want to commit the callnum change if there are other commits + context.commit_pending = False + + # record the login data + self.hostip = hostip + self.callnum = callnum + self.user_id = session_data['user_id'] + self.authtype = session_data['authtype'] + self.master = session_data['master'] + self.session_data = session_data + self.user_data = user_data + self.logged_in = True + + def __getattr__(self, name): + # grab perm and groups data on the fly + if name == 'perms': + if self._perms is None: + # in a dict for quicker lookup + self._perms = dict([[name, 1] for name in get_user_perms(self.user_id)]) + return self._perms + elif name == 'groups': + if self._groups is None: + self._groups = get_user_groups(self.user_id) + return self._groups + elif name == 'host_id': + if self._host_id == '': + self._host_id = self._getHostId() + return self._host_id + else: + raise AttributeError("%s" % name) + + def __str__(self): + # convenient display for debugging + if not self.logged_in: + s = "session: not logged in" + else: + s = "session %d: %r" % (self.id, self.__dict__) + if self.message: + s += " (%s)" % self.message + return s + + def validate(self): + if self.lockerror: + raise koji.AuthLockError(self.lockerror) + return True + + def get_remote_ip(self, override=None): + if not context.opts['CheckClientIP']: + return '-' + elif override is not None: + return override + else: + hostip = context.environ['REMOTE_ADDR'] + # XXX - REMOTE_ADDR not promised by wsgi spec + if hostip == '127.0.0.1': + hostip = socket.gethostbyname(socket.gethostname()) + return hostip + + def checkLoginAllowed(self, user_id): + """Verify that the user is allowed to login""" + query = QueryProcessor(tables=['users'], columns=['name', 'usertype', 'status'], + clauses=['id = %(user_id)i'], values={'user_id': user_id}) + result = query.executeOne(strict=False) + if not result: + raise koji.AuthError('invalid user_id: %s' % user_id) + + if result['status'] != koji.USER_STATUS['NORMAL']: + raise koji.AuthError('logins by %s are not allowed' % result['name']) + + def login(self, user, password, opts=None, renew=False, exclusive=False): + """create a login session""" + if opts is None: + opts = {} + if not isinstance(password, str) or len(password) == 0: + raise koji.AuthError('invalid username or password') + if self.logged_in: + raise koji.AuthError("Already logged in") + hostip = self.get_remote_ip(override=opts.get('hostip')) + + # check passwd + query = QueryProcessor(tables=['users'], columns=['id'], + clauses=['name = %(user)s', 'password = %(password)s'], + values={'user': user, 'password': password}) + user_id = query.singleValue(strict=False) + if not user_id: + raise koji.AuthError('invalid username or password') + + self.checkLoginAllowed(user_id) + + # create session and return + sinfo = self.createSession(user_id, hostip, koji.AUTHTYPES['NORMAL'], renew=renew) + if sinfo and exclusive and not self.exclusive: + self.makeExclusive() + context.cnx.commit() + return sinfo + + def getConnInfo(self): + """Return a tuple containing connection information + in the following format: + (local ip addr, local port, remote ip, remote port)""" + # For some reason req.connection.{local,remote}_addr contain port info, + # but no IP info. Use req.connection.{local,remote}_ip for that instead. + # See: http://lists.planet-lab.org/pipermail/devel-community/2005-June/001084.html + # local_ip seems to always be set to the same value as remote_ip, + # so get the local ip via a different method + local_ip = socket.gethostbyname(context.environ['SERVER_NAME']) + remote_ip = context.environ['REMOTE_ADDR'] + # XXX - REMOTE_ADDR not promised by wsgi spec + + # it appears that calling setports() with *any* value results in authentication + # failing with "Incorrect net address", so return 0 (which prevents + # python-krbV from calling setports()) + local_port = 0 + remote_port = 0 + + return (local_ip, local_port, remote_ip, remote_port) + + def sslLogin(self, proxyuser=None, proxyauthtype=None, renew=False, exclusive=None): + + """Login into brew via SSL. proxyuser name can be specified and if it is + allowed in the configuration file then connection is allowed to login as + that user. By default we assume that proxyuser is coming via same + authentication mechanism but proxyauthtype can be set to koji.AUTHTYPE['*'] + value for different handling. Typical case is proxying kerberos user via + web ui which itself is authenticated via SSL certificate. (See kojiweb + for usage). + + proxyauthtype is working only if AllowProxyAuthType option is set to + 'On' in the hub.conf + """ + if self.logged_in: + raise koji.AuthError("Already logged in") + + # we use REMOTE_USER to identify user + if context.environ.get('REMOTE_USER'): + # it is kerberos principal rather than user's name. + username = context.environ.get('REMOTE_USER') + client_dn = username + authtype = koji.AUTHTYPES['GSSAPI'] + else: + if context.environ.get('SSL_CLIENT_VERIFY') != 'SUCCESS': + raise koji.AuthError('could not verify client: %s' % + context.environ.get('SSL_CLIENT_VERIFY')) + + name_dn_component = context.opts.get('DNUsernameComponent', 'CN') + username = context.environ.get('SSL_CLIENT_S_DN_%s' % name_dn_component) + if not username: + raise koji.AuthError( + 'unable to get user information (%s) from client certificate' % + name_dn_component) + client_dn = context.environ.get('SSL_CLIENT_S_DN') + authtype = koji.AUTHTYPES['SSL'] + + if proxyuser: + if authtype == koji.AUTHTYPES['GSSAPI']: + delimiter = ',' + proxy_opt = 'ProxyPrincipals' + else: + delimiter = '|' + proxy_opt = 'ProxyDNs' + proxy_dns = [dn.strip() for dn in context.opts.get(proxy_opt, '').split(delimiter)] + + if client_dn in proxy_dns: + # the user authorized to login other users + username = proxyuser + else: + raise koji.AuthError('%s is not authorized to login other users' % client_dn) + + # in this point we can continue with proxied user in same way as if it is not proxied + if proxyauthtype is not None: + if not context.opts['AllowProxyAuthType'] and authtype != proxyauthtype: + raise koji.AuthError("Proxy must use same auth mechanism as hub (behaviour " + "can be overriden via AllowProxyAuthType hub option)") + if proxyauthtype not in (koji.AUTHTYPES['GSSAPI'], koji.AUTHTYPES['SSL']): + raise koji.AuthError( + "Proxied authtype %s is not valid for sslLogin" % proxyauthtype) + authtype = proxyauthtype + + if authtype == koji.AUTHTYPES['GSSAPI'] and '@' in username: + user_id = self.getUserIdFromKerberos(username) + else: + user_id = self.getUserId(username) + if not user_id: + if context.opts.get('LoginCreatesUser'): + if authtype == koji.AUTHTYPES['GSSAPI'] and '@' in username: + user_id = self.createUserFromKerberos(username) + else: + user_id = self.createUser(username) + else: + raise koji.AuthError('Unknown user: %s' % username) + + self.checkLoginAllowed(user_id) + + hostip = self.get_remote_ip() + + sinfo = self.createSession(user_id, hostip, authtype, renew=renew) + if sinfo and exclusive and not self.exclusive: + self.makeExclusive() + return sinfo + + def makeExclusive(self, force=False): + """Make this session exclusive""" + if self.master is not None: + raise koji.GenericError("subsessions cannot become exclusive") + if self.exclusive: + # shouldn't happen + raise koji.GenericError("session is already exclusive") + user_id = self.user_id + session_id = self.id + # acquire a row lock on the user entry + query = QueryProcessor(tables=['users'], columns=['id'], clauses=['id=%(user_id)s'], + values={'user_id': user_id}, opts={'rowlock': True}) + query.execute() + # check that no other sessions for this user are exclusive (including expired) + query = QueryProcessor(tables=['sessions'], columns=['id'], + clauses=['user_id=%(user_id)s', 'closed = FALSE', + 'exclusive = TRUE'], + values={'user_id': user_id}, opts={'rowlock': True}) + excl_id = query.singleValue(strict=False) + if excl_id: + if force: + # close the previous exclusive sessions and try again + update = UpdateProcessor('sessions', + data={'expired': True, 'exclusive': None, 'closed': True}, + clauses=['id=%(excl_id)s'], values={'excl_id': excl_id},) + update.execute() + else: + raise koji.AuthLockError("Cannot get exclusive session") + # mark this session exclusive + update = UpdateProcessor('sessions', data={'exclusive': True}, + clauses=['id=%(session_id)s'], values={'session_id': session_id}) + update.execute() + context.cnx.commit() + + def makeShared(self): + """Drop out of exclusive mode""" + session_id = self.id + update = UpdateProcessor('sessions', data={'exclusive': None}, + clauses=['id=%(session_id)s'], values={'session_id': session_id}) + update.execute() + context.cnx.commit() + + def logout(self, session_id=None): + """close a login session""" + if not self.logged_in: + # XXX raise an error? + raise koji.AuthError("Not logged in") + + if session_id: + if not context.session.hasPerm('admin'): + query = QueryProcessor(tables=['sessions'], columns=['id'], + clauses=['user_id = %(user_id)i', 'id = %(session_id)s'], + values={'user_id': self.user_id, 'session_id': session_id}) + if not query.singleValue(): + raise koji.ActionNotAllowed('only admins or owner may logout other session') + ses_id = session_id + else: + ses_id = self.id + update = UpdateProcessor('sessions', + data={'expired': True, 'exclusive': None, 'closed': True}, + clauses=['id = %(id)i OR master = %(id)i'], + values={'id': ses_id}) + update.execute() + context.cnx.commit() + if not session_id: + self.logged_in = False + + def logoutChild(self, session_id): + """close a subsession""" + if not self.logged_in: + # XXX raise an error? + raise koji.AuthError("Not logged in") + update = UpdateProcessor('sessions', + data={'expired': True, 'exclusive': None, 'closed': True}, + clauses=['id = %(session_id)i', 'master = %(master)i'], + values={'session_id': session_id, 'master': self.id}) + update.execute() + context.cnx.commit() + + def createSession(self, user_id, hostip, authtype, master=None, renew=False): + """Create a new session for the given user. + + Return a map containing the session-id and session-key. + If master is specified, create a subsession + """ + # generate a random key + alnum = string.ascii_letters + string.digits + key = "%s-%s" % (user_id, + ''.join([random.choice(alnum) for x in range(1, 20)])) + # use sha? sha.new(phrase).hexdigest() + + if renew and self.id is not None: + # just update key + session_id = self.id + self.key = key + if self.master: + # check if master session died meanwhile (expired is ok) + query = QueryProcessor(tables=['sessions'], + clauses=['id = %(master_id)d', 'closed IS FALSE'], + values={'master_id': self.master}, + opts={'countOnly': True}) + if query.executeOne() == 0: + return None + + update = UpdateProcessor('sessions', + clauses=['id=%(id)i'], + rawdata={'update_time': 'NOW()'}, + data={'key': self.key, 'expired': False}, + values={'id': self.id}) + update.execute() + else: + # get a session id + session_id = nextval('sessions_id_seq') + # add session id to database + insert = InsertProcessor('sessions', + data={'id': session_id, 'user_id': user_id, 'key': key, + 'hostip': hostip, 'authtype': authtype, + 'master': master}) + insert.execute() + context.cnx.commit() + + # return session info + return { + 'session-id': session_id, + 'session-key': key, + 'header-auth': True, # signalize to client to use new session handling in 1.30 + } + + def subsession(self): + "Create a subsession" + if not self.logged_in: + raise koji.AuthError("Not logged in") + master = self.master + if master is None: + master = self.id + return self.createSession(self.user_id, self.hostip, self.authtype, master=master) + + def getPerms(self): + if not self.logged_in: + return [] + return to_list(self.perms.keys()) + + def hasPerm(self, name): + if not self.logged_in: + return False + return name in self.perms + + def assertPerm(self, name): + if not self.hasPerm(name) and not self.hasPerm('admin'): + msg = "%s permission required" % name + if self.logged_in: + msg += ' (logged in as %s)' % self.user_data['name'] + else: + msg += ' (user not logged in)' + raise koji.ActionNotAllowed(msg) + + def assertLogin(self): + if not self.logged_in: + raise koji.ActionNotAllowed("you must be logged in for this operation") + + def hasGroup(self, group_id): + if not self.logged_in: + return False + # groups indexed by id + return group_id in self.groups + + def isUser(self, user_id): + if not self.logged_in: + return False + return (self.user_id == user_id or self.hasGroup(user_id)) + + def assertUser(self, user_id): + if not self.isUser(user_id) and not self.hasPerm('admin'): + raise koji.ActionNotAllowed("not owner") + + def _getHostId(self): + '''Using session data, find host id (if there is one)''' + if self.user_id is None: + return None + query = QueryProcessor(tables=['host'], columns=['id'], clauses=['user_id = %(uid)d'], + values={'uid': self.user_id}) + return query.singleValue(strict=False) + + def getHostId(self): + # for compatibility + return self.host_id + + def getUserId(self, username): + """Return the user ID associated with a particular username. If no user + with the given username if found, return None.""" + query = QueryProcessor(tables=['users'], columns=['id'], clauses=['name = %(username)s'], + values={'username': username}) + return query.singleValue(strict=False) + + def getUserIdFromKerberos(self, krb_principal): + """Return the user ID associated with a particular Kerberos principal. + If no user with the given princpal if found, return None.""" + self.checkKrbPrincipal(krb_principal) + query = QueryProcessor(tables=['users'], columns=['id'], + joins=['user_krb_principals ON ' + 'users.id = user_krb_principals.user_id'], + clauses=['krb_principal = %(krb_principal)s'], + values={'krb_principal': krb_principal}) + return query.singleValue(strict=False) + + def createUser(self, name, usertype=None, status=None, krb_principal=None, + krb_princ_check=True): + """ + Create a new user, using the provided values. + Return the user_id of the newly-created user. + """ + if not name: + raise koji.GenericError('a user must have a non-empty name') + + if usertype is None: + usertype = koji.USERTYPES['NORMAL'] + elif not koji.USERTYPES.get(usertype): + raise koji.GenericError('invalid user type: %s' % usertype) + + if status is None: + status = koji.USER_STATUS['NORMAL'] + elif not koji.USER_STATUS.get(status): + raise koji.GenericError('invalid status: %s' % status) + + # check if krb_principal is allowed + if krb_princ_check: + self.checkKrbPrincipal(krb_principal) + + user_id = nextval('users_id_seq') + + insert = InsertProcessor('users', + data={'id': user_id, 'name': name, 'usertype': usertype, + 'status': status}) + insert.execute() + if krb_principal: + insert = InsertProcessor('user_krb_principals', + data={'user_id': user_id, 'krb_principal': krb_principal}) + insert.execute() + context.cnx.commit() + + return user_id + + def setKrbPrincipal(self, name, krb_principal, krb_princ_check=True): + if krb_princ_check: + self.checkKrbPrincipal(krb_principal) + if isinstance(name, six.integer_types): + clauses = ['id = %(name)i'] + else: + clauses = ['name = %(name)s'] + query = QueryProcessor(tables=['users'], columns=['id'], clauses=clauses, + values={'name': name}) + user_id = query.singleValue(strict=False) + if not user_id: + context.cnx.rollback() + raise koji.AuthError('No such user: %s' % name) + insert = InsertProcessor('user_krb_principals', + data={'user_id': user_id, 'krb_principal': krb_principal}) + insert.execute() + context.cnx.commit() + return user_id + + def removeKrbPrincipal(self, name, krb_principal): + clauses = ['krb_principal = %(krb_principal)s'] + if isinstance(name, six.integer_types): + clauses.extend(['id = %(name)i']) + else: + clauses.extend(['name = %(name)s']) + query = QueryProcessor(tables=['users'], columns=['id'], + joins=['user_krb_principals ' + 'ON users.id = user_krb_principals.user_id'], + clauses=clauses, + values={'krb_principal': krb_principal, 'name': name}) + user_id = query.singleValue(strict=False) + if not user_id: + context.cnx.rollback() + raise koji.AuthError( + 'cannot remove Kerberos Principal:' + ' %(krb_principal)s with user %(name)s' % locals()) + cursor = context.cnx.cursor() + delete = DeleteProcessor(table='user_krb_principals', + clauses=['user_id = %(user_id)i', + 'krb_principal = %(krb_principal)s'], + values={'user_id': user_id, 'krb_principal': krb_principal}) + delete.execute() + context.cnx.commit() + return user_id + + def createUserFromKerberos(self, krb_principal): + """Create a new user, based on the Kerberos principal. Their + username will be everything before the "@" in the principal. + Return the ID of the newly created user.""" + atidx = krb_principal.find('@') + if atidx == -1: + raise koji.AuthError('invalid Kerberos principal: %s' % krb_principal) + user_name = krb_principal[:atidx] + + # check if user already exists + query = QueryProcessor(tables=['users'], columns=['id', 'krb_principal'], + joins=['LEFT JOIN user_krb_principals ON ' + 'users.id = user_krb_principals.user_id'], + clauses=['name = %(user_name)s'], + values={'user_name': user_name}) + r = query.execute() + if not r: + return self.createUser(user_name, krb_principal=krb_principal, + krb_princ_check=False) + else: + existing_user_krb_princs = [row['krb_principal'] for row in r] + if krb_principal in existing_user_krb_princs: + # do not set Kerberos principal if it already exists + return r[0]['id'] + return self.setKrbPrincipal(user_name, krb_principal, krb_princ_check=False) + + def checkKrbPrincipal(self, krb_principal): + """Check if the Kerberos principal is allowed""" + if krb_principal is None: + return + allowed_realms = context.opts.get('AllowedKrbRealms', '*') + if allowed_realms == '*': + return + allowed_realms = re.split(r'\s*,\s*', allowed_realms) + atidx = krb_principal.find('@') + if atidx == -1 or atidx == len(krb_principal) - 1: + raise koji.AuthError( + 'invalid Kerberos principal: %s' % krb_principal) + realm = krb_principal[atidx + 1:] + if realm not in allowed_realms: + raise koji.AuthError( + "Kerberos principal's realm: %s is not allowed" % realm) + + +def get_user_groups(user_id): + """Get user groups + + returns a dictionary where the keys are the group ids and the values + are the group names""" + t_group = koji.USERTYPES['GROUP'] + query = QueryProcessor(tables=['user_groups'], columns=['group_id', 'name'], + clauses=['active = TRUE', 'users.usertype=%(t_group)i', + 'user_id=%(user_id)i'], + joins=['users ON group_id = users.id'], + values={'t_group': t_group, 'user_id': user_id}) + return query.execute() + + +def get_user_perms(user_id): + query = QueryProcessor(tables=['user_perms'], columns=['name'], + clauses=['active = TRUE', 'user_id=%(user_id)s'], + joins=['permissions ON perm_id = permissions.id'], + values={'user_id': user_id}) + result = query.execute() + return [r['name'] for r in result] + + +def get_user_data(user_id): + query = QueryProcessor(tables=['users'], columns=['name', 'status', 'usertype'], + clauses=['id=%(user_id)s'], values={'user_id': user_id}) + return query.executeOne(strict=False) + + +def login(*args, **opts): + """Create a login session with plain user/password credentials. + + :param str user: username + :param str password: password + :param dict opts: curently can contain only 'host_ip' key for overriding client IP address + + :returns dict: session info + """ + + return context.session.login(*args, **opts) + + +def sslLogin(*args, **opts): + """Login via SSL certificate + + :param str proxyuser: proxy username + :returns dict: session info + """ + return context.session.sslLogin(*args, **opts) + + +def logout(session_id=None): + """expire a login session""" + return context.session.logout(session_id) + + +def subsession(): + """Create a subsession""" + return context.session.subsession() + + +def logoutChild(session_id): + """expire a subsession + + :param int subsession_id: subsession ID (for current session) + """ + return context.session.logoutChild(session_id) + + +def exclusiveSession(*args, **opts): + """Make this session exclusive""" + return context.session.makeExclusive(*args, **opts) + + +def sharedSession(): + """Drop out of exclusive mode""" + return context.session.makeShared() diff --git a/kojihub/db.py b/kojihub/db.py new file mode 100644 index 0000000..478c30f --- /dev/null +++ b/kojihub/db.py @@ -0,0 +1,924 @@ +# python library + +# db utilities for koji +# Copyright (c) 2005-2014 Red Hat, Inc. +# +# Koji is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; +# version 2.1 of the License. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this software; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: +# Mike McLean + + +from __future__ import absolute_import + +import logging +import koji +import os +# import psycopg2.extensions +# # don't convert timestamp fields to DateTime objects +# del psycopg2.extensions.string_types[1114] +# del psycopg2.extensions.string_types[1184] +# del psycopg2.extensions.string_types[1082] +# del psycopg2.extensions.string_types[1083] +# del psycopg2.extensions.string_types[1266] +import re +import sys +import time +import traceback + +import psycopg2 + +import koji.context +context = koji.context.context + + +POSITIONAL_RE = re.compile(r'%[a-z]') +NAMED_RE = re.compile(r'%\(([^\)]+)\)[a-z]') + +## Globals ## +_DBopts = None +# A persistent connection to the database. +# A new connection will be created whenever +# Apache forks a new worker, and that connection +# will be used to service all requests handled +# by that worker. +# This probably doesn't need to be a ThreadLocal +# since Apache is not using threading, +# but play it safe anyway. +_DBconn = koji.context.ThreadLocal() + +logger = logging.getLogger('koji.db') + + +class DBWrapper: + def __init__(self, cnx): + self.cnx = cnx + + def __getattr__(self, key): + if not self.cnx: + raise Exception('connection is closed') + return getattr(self.cnx, key) + + def cursor(self, *args, **kw): + if not self.cnx: + raise Exception('connection is closed') + return CursorWrapper(self.cnx.cursor(*args, **kw)) + + def close(self): + # Rollback any uncommitted changes and clear the connection so + # this DBWrapper is no longer usable after close() + if not self.cnx: + raise Exception('connection is closed') + self.cnx.cursor().execute('ROLLBACK') + # We do this rather than cnx.rollback to avoid opening a new transaction + # If our connection gets recycled cnx.rollback will be called then. + self.cnx = None + + +class CursorWrapper: + def __init__(self, cursor): + self.cursor = cursor + self.logger = logging.getLogger('koji.db') + + def __getattr__(self, key): + return getattr(self.cursor, key) + + def _timed_call(self, method, args, kwargs): + start = time.time() + ret = getattr(self.cursor, method)(*args, **kwargs) + self.logger.debug("%s operation completed in %.4f seconds", method, time.time() - start) + return ret + + def fetchone(self, *args, **kwargs): + return self._timed_call('fetchone', args, kwargs) + + def fetchall(self, *args, **kwargs): + return self._timed_call('fetchall', args, kwargs) + + def quote(self, operation, parameters): + if hasattr(self.cursor, "mogrify"): + quote = self.cursor.mogrify + else: + def quote(a, b): + return a % b + try: + return quote(operation, parameters) + except Exception: + self.logger.exception( + 'Unable to quote query:\n%s\nParameters: %s', operation, parameters) + return "INVALID QUERY" + + def preformat(self, sql, params): + """psycopg2 requires all variable placeholders to use the string (%s) datatype, + regardless of the actual type of the data. Format the sql string to be compliant. + It also requires IN parameters to be in tuple rather than list format.""" + sql = POSITIONAL_RE.sub(r'%s', sql) + sql = NAMED_RE.sub(r'%(\1)s', sql) + if isinstance(params, dict): + for name, value in params.items(): + if isinstance(value, list): + params[name] = tuple(value) + else: + if isinstance(params, tuple): + params = list(params) + for i, item in enumerate(params): + if isinstance(item, list): + params[i] = tuple(item) + return sql, params + + def execute(self, operation, parameters=(), log_errors=True): + debug = self.logger.isEnabledFor(logging.DEBUG) + operation, parameters = self.preformat(operation, parameters) + if debug: + self.logger.debug(self.quote(operation, parameters)) + start = time.time() + try: + ret = self.cursor.execute(operation, parameters) + except Exception: + if log_errors: + self.logger.error('Query failed. Query was: %s', self.quote(operation, parameters)) + raise + if debug: + self.logger.debug("Execute operation completed in %.4f seconds", time.time() - start) + return ret + + +## Functions ## +def provideDBopts(**opts): + global _DBopts + if _DBopts is None: + _DBopts = dict([i for i in opts.items() if i[1] is not None]) + + +def setDBopts(**opts): + global _DBopts + _DBopts = opts + + +def getDBopts(): + return _DBopts + + +def connect(): + logger = logging.getLogger('koji.db') + global _DBconn + if hasattr(_DBconn, 'conn'): + # Make sure the previous transaction has been + # closed. This is safe to call multiple times. + conn = _DBconn.conn + try: + # Under normal circumstances, the last use of this connection + # will have issued a raw ROLLBACK to close the transaction. To + # avoid 'no transaction in progress' warnings (depending on postgres + # configuration) we open a new one here. + # Should there somehow be a transaction in progress, a second + # BEGIN will be a harmless no-op, though there may be a warning. + conn.cursor().execute('BEGIN') + conn.rollback() + return DBWrapper(conn) + except psycopg2.Error: + del _DBconn.conn + # create a fresh connection + opts = _DBopts + if opts is None: + opts = {} + try: + if 'dsn' in opts: + conn = psycopg2.connect(dsn=opts['dsn']) + else: + conn = psycopg2.connect(**opts) + conn.set_client_encoding('UTF8') + except Exception: + logger.error(''.join(traceback.format_exception(*sys.exc_info()))) + raise + # XXX test + # return conn + _DBconn.conn = conn + + return DBWrapper(conn) + + +def _dml(operation, values, log_errors=True): + """Run an insert, update, or delete. Return number of rows affected + If log is False, errors will not be logged. It makes sense only for + queries which are expected to fail (LOCK NOWAIT) + """ + c = context.cnx.cursor() + c.execute(operation, values, log_errors=log_errors) + ret = c.rowcount + logger.debug("Operation affected %s row(s)", ret) + c.close() + context.commit_pending = True + return ret + + +def _fetchMulti(query, values): + """Run the query and return all rows""" + c = context.cnx.cursor() + c.execute(query, values) + results = c.fetchall() + c.close() + return results + + +def _fetchSingle(query, values, strict=False): + """Run the query and return a single row + + If strict is true, raise an error if the query returns more or less than + one row.""" + results = _fetchMulti(query, values) + numRows = len(results) + if numRows == 0: + if strict: + raise koji.GenericError('query returned no rows') + else: + return None + elif strict and numRows > 1: + raise koji.GenericError('multiple rows returned for a single row query') + else: + return results[0] + + +def _singleValue(query, values=None, strict=True): + """Perform a query that returns a single value. + + Note that unless strict is True a return value of None could mean either + a single NULL value or zero rows returned.""" + if values is None: + values = {} + row = _fetchSingle(query, values, strict) + if row: + if strict and len(row) > 1: + raise koji.GenericError('multiple fields returned for a single value query') + return row[0] + else: + # don't need to check strict here, since that was already handled by _singleRow() + return None + + +def _multiRow(query, values, fields): + """Return all rows from "query". Named query parameters + can be specified using the "values" map. Results will be returned + as a list of maps. Each map in the list will have a key for each + element in the "fields" list. If there are no results, an empty + list will be returned.""" + return [dict(zip(fields, row)) for row in _fetchMulti(query, values)] + + +def _singleRow(query, values, fields, strict=False): + """Return a single row from "query". Named parameters can be + specified using the "values" map. The result will be returned as + as map. The map will have a key for each element in the "fields" + list. If more than one row is returned and "strict" is true, a + GenericError will be raised. If no rows are returned, and "strict" + is True, a GenericError will be raised. Otherwise None will be + returned.""" + row = _fetchSingle(query, values, strict) + if row: + return dict(zip(fields, row)) + else: + # strict enforced by _fetchSingle + return None + + +def get_event(): + """Get an event id for this transaction + + We cache the result in context, so subsequent calls in the same transaction will + get the same event. + + This cache is cleared between the individual calls in a multicall. + See: https://pagure.io/koji/pull-request/74 + """ + if hasattr(context, 'event_id'): + return context.event_id + event_id = _singleValue("SELECT get_event()") + context.event_id = event_id + return event_id + + +def nextval(sequence): + """Get the next value for the given sequence""" + data = {'sequence': sequence} + return _singleValue("SELECT nextval(%(sequence)s)", data, strict=True) + + +def currval(sequence): + """Get the current value for the given sequence""" + data = {'sequence': sequence} + return _singleValue("SELECT currval(%(sequence)s)", data, strict=True) + + +class Savepoint(object): + + def __init__(self, name): + self.name = name + _dml("SAVEPOINT %s" % name, {}) + + def rollback(self): + _dml("ROLLBACK TO SAVEPOINT %s" % self.name, {}) + + +class InsertProcessor(object): + """Build an insert statement + + table - the table to insert into + data - a dictionary of data to insert (keys = row names) + rawdata - data to insert specified as sql expressions rather than python values + + does not support query inserts of "DEFAULT VALUES" + """ + + def __init__(self, table, data=None, rawdata=None): + self.table = table + self.data = {} + if data: + self.data.update(data) + self.rawdata = {} + if rawdata: + self.rawdata.update(rawdata) + + def __str__(self): + if not self.data and not self.rawdata: + return "-- incomplete update: no assigns" + parts = ['INSERT INTO %s ' % self.table] + columns = sorted(list(self.data.keys()) + list(self.rawdata.keys())) + parts.append("(%s) " % ', '.join(columns)) + values = [] + for key in columns: + if key in self.data: + values.append("%%(%s)s" % key) + else: + values.append("(%s)" % self.rawdata[key]) + parts.append("VALUES (%s)" % ', '.join(values)) + return ''.join(parts) + + def __repr__(self): + return "" % vars(self) + + def set(self, **kwargs): + """Set data via keyword args""" + self.data.update(kwargs) + + def rawset(self, **kwargs): + """Set rawdata via keyword args""" + self.rawdata.update(kwargs) + + def make_create(self, event_id=None, user_id=None): + if event_id is None: + event_id = get_event() + if user_id is None: + context.session.assertLogin() + user_id = context.session.user_id + self.data['create_event'] = event_id + self.data['creator_id'] = user_id + + def dup_check(self): + """Check to see if the insert duplicates an existing row""" + if self.rawdata: + logger.warning("Can't perform duplicate check") + return None + data = self.data.copy() + if 'create_event' in self.data: + # versioned table + data['active'] = True + del data['create_event'] + del data['creator_id'] + clauses = ["%s = %%(%s)s" % (k, k) for k in data] + query = QueryProcessor(columns=list(data.keys()), tables=[self.table], + clauses=clauses, values=data) + if query.execute(): + return True + return False + + def execute(self): + return _dml(str(self), self.data) + + +class UpdateProcessor(object): + """Build an update statement + + table - the table to insert into + data - a dictionary of data to insert (keys = row names) + rawdata - data to insert specified as sql expressions rather than python values + clauses - a list of where clauses which will be ANDed together + values - dict of values used in clauses + + does not support the FROM clause + """ + + def __init__(self, table, data=None, rawdata=None, clauses=None, values=None): + self.table = table + self.data = {} + if data: + self.data.update(data) + self.rawdata = {} + if rawdata: + self.rawdata.update(rawdata) + self.clauses = [] + if clauses: + self.clauses.extend(clauses) + self.values = {} + if values: + self.values.update(values) + + def __str__(self): + if not self.data and not self.rawdata: + return "-- incomplete update: no assigns" + parts = ['UPDATE %s SET ' % self.table] + assigns = ["%s = %%(data.%s)s" % (key, key) for key in self.data] + assigns.extend(["%s = (%s)" % (key, self.rawdata[key]) for key in self.rawdata]) + parts.append(', '.join(sorted(assigns))) + if self.clauses: + parts.append('\nWHERE ') + parts.append(' AND '.join(["( %s )" % c for c in sorted(self.clauses)])) + return ''.join(parts) + + def __repr__(self): + return "" % vars(self) + + def get_values(self): + """Returns unified values dict, including data""" + ret = {} + ret.update(self.values) + for key in self.data: + ret["data." + key] = self.data[key] + return ret + + def set(self, **kwargs): + """Set data via keyword args""" + self.data.update(kwargs) + + def rawset(self, **kwargs): + """Set rawdata via keyword args""" + self.rawdata.update(kwargs) + + def make_revoke(self, event_id=None, user_id=None): + """Add standard revoke options to the update""" + if event_id is None: + event_id = get_event() + if user_id is None: + context.session.assertLogin() + user_id = context.session.user_id + self.data['revoke_event'] = event_id + self.data['revoker_id'] = user_id + self.rawdata['active'] = 'NULL' + self.clauses.append('active = TRUE') + + def execute(self): + return _dml(str(self), self.get_values()) + + +class DeleteProcessor(object): + """Build an delete statement + + table - the table to delete + clauses - a list of where clauses which will be ANDed together + values - dict of values used in clauses + """ + + def __init__(self, table, clauses=None, values=None): + self.table = table + self.clauses = [] + if clauses: + self.clauses.extend(clauses) + self.values = {} + if values: + self.values.update(values) + + def __str__(self): + parts = ['DELETE FROM %s ' % self.table] + if self.clauses: + parts.append('\nWHERE ') + parts.append(' AND '.join(["( %s )" % c for c in sorted(self.clauses)])) + return ''.join(parts) + + def __repr__(self): + return "" % vars(self) + + def get_values(self): + """Returns unified values dict, including data""" + ret = {} + ret.update(self.values) + return ret + + def execute(self): + return _dml(str(self), self.get_values()) + + +class QueryProcessor(object): + """ + Build a query from its components. + - columns, aliases, tables: lists of the column names to retrieve, + the tables to retrieve them from, and the key names to use when + returning values as a map, respectively + - joins: a list of joins in the form 'table1 ON table1.col1 = table2.col2', 'JOIN' will be + prepended automatically; if extended join syntax (LEFT, OUTER, etc.) is required, + it can be specified, and 'JOIN' will not be prepended + - clauses: a list of where clauses in the form 'table1.col1 OPER table2.col2-or-variable'; + each clause will be surrounded by parentheses and all will be AND'ed together + - values: the map that will be used to replace any substitution expressions in the query + - transform: a function that will be called on each row (not compatible with + countOnly or singleValue) + - opts: a map of query options; currently supported options are: + countOnly: if True, return an integer indicating how many results would have been + returned, rather than the actual query results + order: a column or alias name to use in the 'ORDER BY' clause + offset: an integer to use in the 'OFFSET' clause + limit: an integer to use in the 'LIMIT' clause + asList: if True, return results as a list of lists, where each list contains the + column values in query order, rather than the usual list of maps + rowlock: if True, use "FOR UPDATE" to lock the queried rows + group: a column or alias name to use in the 'GROUP BY' clause + (controlled by enable_group) + - enable_group: if True, opts.group will be enabled + """ + + iterchunksize = 1000 + + def __init__(self, columns=None, aliases=None, tables=None, + joins=None, clauses=None, values=None, transform=None, + opts=None, enable_group=False): + self.columns = columns + self.aliases = aliases + if columns and aliases: + if len(columns) != len(aliases): + raise Exception('column and alias lists must be the same length') + # reorder + alias_table = sorted(zip(aliases, columns)) + self.aliases = [x[0] for x in alias_table] + self.columns = [x[1] for x in alias_table] + self.colsByAlias = dict(alias_table) + else: + self.colsByAlias = {} + if columns: + self.columns = sorted(columns) + if aliases: + self.aliases = sorted(aliases) + self.tables = tables + self.joins = joins + if clauses: + self.clauses = sorted(clauses) + else: + self.clauses = clauses + self.cursors = 0 + if values: + self.values = values + else: + self.values = {} + self.transform = transform + if opts: + self.opts = opts + else: + self.opts = {} + self.enable_group = enable_group + self.logger = logging.getLogger('koji.db') + + def countOnly(self, count): + self.opts['countOnly'] = count + + def __str__(self): + query = \ + """ +SELECT %(col_str)s + FROM %(table_str)s +%(join_str)s +%(clause_str)s + %(group_str)s + %(order_str)s +%(offset_str)s + %(limit_str)s +""" + if self.opts.get('countOnly'): + if self.opts.get('offset') \ + or self.opts.get('limit') \ + or (self.enable_group and self.opts.get('group')): + # If we're counting with an offset and/or limit, we need + # to wrap the offset/limited query and then count the results, + # rather than trying to offset/limit the single row returned + # by count(*). Because we're wrapping the query, we don't care + # about the column values. + col_str = '1' + else: + col_str = 'count(*)' + else: + col_str = self._seqtostr(self.columns) + table_str = self._seqtostr(self.tables, sort=True) + join_str = self._joinstr() + clause_str = self._seqtostr(self.clauses, sep=')\n AND (') + if clause_str: + clause_str = ' WHERE (' + clause_str + ')' + if self.enable_group: + group_str = self._group() + else: + group_str = '' + order_str = self._order() + offset_str = self._optstr('offset') + limit_str = self._optstr('limit') + + query = query % locals() + if self.opts.get('countOnly') and \ + (self.opts.get('offset') or + self.opts.get('limit') or + (self.enable_group and self.opts.get('group'))): + query = 'SELECT count(*)\nFROM (' + query + ') numrows' + if self.opts.get('rowlock'): + query += '\n FOR UPDATE' + return query + + def __repr__(self): + return '' % \ + (self.columns, self.aliases, self.tables, self.joins, self.clauses, self.values, + self.opts) + + def _seqtostr(self, seq, sep=', ', sort=False): + if seq: + if sort: + seq = sorted(seq) + return sep.join(seq) + else: + return '' + + def _joinstr(self): + if not self.joins: + return '' + result = '' + for join in self.joins: + if result: + result += '\n' + if re.search(r'\bjoin\b', join, re.IGNORECASE): + # The join clause already contains the word 'join', + # so don't prepend 'JOIN' to it + result += ' ' + join + else: + result += ' JOIN ' + join + return result + + def _order(self): + # Don't bother sorting if we're just counting + if self.opts.get('countOnly'): + return '' + order_opt = self.opts.get('order') + if order_opt: + order_exprs = [] + for order in order_opt.split(','): + if order.startswith('-'): + order = order[1:] + direction = ' DESC' + else: + direction = '' + # Check if we're ordering by alias first + orderCol = self.colsByAlias.get(order) + if orderCol: + pass + elif order in self.columns: + orderCol = order + else: + raise Exception('Invalid order: ' + order) + order_exprs.append(orderCol + direction) + return 'ORDER BY ' + ', '.join(order_exprs) + else: + return '' + + def _group(self): + group_opt = self.opts.get('group') + if group_opt: + group_exprs = [] + for group in group_opt.split(','): + if group: + group_exprs.append(group) + return 'GROUP BY ' + ', '.join(group_exprs) + else: + return '' + + def _optstr(self, optname): + optval = self.opts.get(optname) + if optval: + return '%s %i' % (optname.upper(), optval) + else: + return '' + + def singleValue(self, strict=True): + # self.transform not applied here + return _singleValue(str(self), self.values, strict=strict) + + def execute(self): + query = str(self) + if self.opts.get('countOnly'): + return _singleValue(query, self.values, strict=True) + elif self.opts.get('asList'): + if self.transform is None: + return _fetchMulti(query, self.values) + else: + # if we're transforming, generate the dicts so the transform can modify + fields = self.aliases or self.columns + data = _multiRow(query, self.values, fields) + data = [self.transform(row) for row in data] + # and then convert back to lists + data = [[row[f] for f in fields] for row in data] + return data + else: + data = _multiRow(query, self.values, (self.aliases or self.columns)) + if self.transform is not None: + data = [self.transform(row) for row in data] + return data + + def iterate(self): + if self.opts.get('countOnly'): + return self.execute() + elif self.opts.get('limit') and self.opts['limit'] < self.iterchunksize: + return self.execute() + else: + fields = self.aliases or self.columns + fields = list(fields) + cname = "qp_cursor_%s_%i_%i" % (id(self), os.getpid(), self.cursors) + self.cursors += 1 + self.logger.debug('Setting up query iterator. cname=%r', cname) + return self._iterate(cname, str(self), self.values.copy(), fields, + self.iterchunksize, self.opts.get('asList')) + + def _iterate(self, cname, query, values, fields, chunksize, as_list=False): + # We pass all this data into the generator so that the iterator works + # from the snapshot when it was generated. Otherwise reuse of the processor + # for similar queries could have unpredictable results. + query = "DECLARE %s NO SCROLL CURSOR FOR %s" % (cname, query) + c = context.cnx.cursor() + c.execute(query, values) + c.close() + try: + query = "FETCH %i FROM %s" % (chunksize, cname) + while True: + if as_list: + if self.transform is None: + buf = _fetchMulti(query, {}) + else: + # if we're transforming, generate the dicts so the transform can modify + buf = _multiRow(query, self.values, fields) + buf = [self.transform(row) for row in buf] + # and then convert back to lists + buf = [[row[f] for f in fields] for row in buf] + else: + buf = _multiRow(query, {}, fields) + if self.transform is not None: + buf = [self.transform(row) for row in buf] + if not buf: + break + for row in buf: + yield row + finally: + c = context.cnx.cursor() + c.execute("CLOSE %s" % cname) + c.close() + + def executeOne(self, strict=False): + results = self.execute() + if isinstance(results, list): + if len(results) > 0: + if strict and len(results) > 1: + raise koji.GenericError('multiple rows returned for a single row query') + return results[0] + elif strict: + raise koji.GenericError('query returned no rows') + else: + return None + return results + + +class BulkInsertProcessor(object): + def __init__(self, table, data=None, columns=None, strict=True, batch=1000): + """Do bulk inserts - it has some limitations compared to + InsertProcessor (no rawset, dup_check). + + set() is replaced with add_record() to avoid confusion + + table - name of the table + data - list of dict per record + columns - list/set of names of used columns - makes sense + mainly with strict=True + strict - if True, all records must contain values for all columns. + if False, missing values will be inserted as NULLs + batch - batch size for inserts (one statement per batch) + """ + + self.table = table + self.data = [] + if columns is None: + self.columns = set() + else: + self.columns = set(columns) + if data is not None: + self.data = data + for row in data: + self.columns |= set(row.keys()) + self.strict = strict + self.batch = batch + + def __str__(self): + if not self.data: + return "-- incomplete insert: no data" + query, params = self._get_insert(self.data) + return query + + def _get_insert(self, data): + """ + Generate one insert statement for the given data + + :param list data: list of rows (dict format) to insert + :returns: (query, params) + """ + + if not data: + # should not happen + raise ValueError('no data for insert') + parts = ['INSERT INTO %s ' % self.table] + columns = sorted(self.columns) + parts.append("(%s) " % ', '.join(columns)) + + prepared_data = {} + values = [] + i = 0 + for row in data: + row_values = [] + for key in columns: + if key in row: + row_key = '%s%d' % (key, i) + row_values.append("%%(%s)s" % row_key) + prepared_data[row_key] = row[key] + elif self.strict: + raise koji.GenericError("Missing value %s in BulkInsert" % key) + else: + row_values.append("NULL") + values.append("(%s)" % ', '.join(row_values)) + i += 1 + parts.append("VALUES %s" % ', '.join(values)) + return ''.join(parts), prepared_data + + def __repr__(self): + return "" % vars(self) + + def add_record(self, **kwargs): + """Set whole record via keyword args""" + if not kwargs: + raise koji.GenericError("Missing values in BulkInsert.add_record") + self.data.append(kwargs) + self.columns |= set(kwargs.keys()) + + def execute(self): + if not self.batch: + self._one_insert(self.data) + else: + for i in range(0, len(self.data), self.batch): + data = self.data[i:i + self.batch] + self._one_insert(data) + + def _one_insert(self, data): + query, params = self._get_insert(data) + _dml(query, params) + + +def _applyQueryOpts(results, queryOpts): + """ + Apply queryOpts to results in the same way QueryProcessor would. + results is a list of maps. + queryOpts is a map which may contain the following fields: + countOnly + order + offset + limit + + Note: + - asList is supported by QueryProcessor but not by this method. + We don't know the original query order, and so don't have a way to + return a useful list. asList should be handled by the caller. + - group is supported by QueryProcessor but not by this method as well. + """ + if queryOpts is None: + queryOpts = {} + if queryOpts.get('order'): + order = queryOpts['order'] + reverse = False + if order.startswith('-'): + order = order[1:] + reverse = True + results.sort(key=lambda o: o[order], reverse=reverse) + if queryOpts.get('offset'): + results = results[queryOpts['offset']:] + if queryOpts.get('limit'): + results = results[:queryOpts['limit']] + if queryOpts.get('countOnly'): + return len(results) + else: + return results diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index d76817d..f7d5a44 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -55,8 +55,6 @@ import rpm from psycopg2._psycopg import IntegrityError import koji -import koji.auth -import koji.db import koji.plugin import koji.policy import koji.rpmdiff @@ -76,7 +74,8 @@ from koji.util import ( multi_fnmatch, safer_move, ) -from koji.db import ( # noqa: F401 +from .auth import get_user_perms, get_user_groups +from .db import ( # noqa: F401 BulkInsertProcessor, DeleteProcessor, InsertProcessor, @@ -1744,7 +1743,7 @@ def check_tag_access(tag_id, user_id=None): if user_id is None: raise koji.GenericError("a user_id is required") user_id = convert_value(user_id, cast=int) - perms = koji.auth.get_user_perms(user_id) + perms = get_user_perms(user_id) override = False if 'admin' in perms: override = True @@ -9628,7 +9627,7 @@ class IsBuildOwnerTest(koji.policy.BaseSimpleTest): return True if owner['usertype'] == koji.USERTYPES['GROUP']: # owner is a group, check to see if user is a member - if owner['id'] in koji.auth.get_user_groups(user['id']): + if owner['id'] in get_user_groups(user['id']): return True # otherwise... return False @@ -9646,7 +9645,7 @@ class UserInGroupTest(koji.policy.BaseSimpleTest): user = policy_get_user(data) if not user: return False - groups = koji.auth.get_user_groups(user['id']) + groups = get_user_groups(user['id']) args = self.str.split()[1:] for group_id, group in groups.items(): for pattern in args: @@ -9668,7 +9667,7 @@ class HasPermTest(koji.policy.BaseSimpleTest): user = policy_get_user(data) if not user: return False - perms = koji.auth.get_user_perms(user['id']) + perms = get_user_perms(user['id']) args = self.str.split()[1:] for perm in perms: for pattern in args: @@ -9811,7 +9810,7 @@ def check_policy(name, data, default='deny', strict=False, force=False): logger.error("Invalid action in policy %s, rule: %s", name, lastrule) if force: user = policy_get_user(data) - if user and 'admin' in koji.auth.get_user_perms(user['id']): + if user and 'admin' in get_user_perms(user['id']): msg = "Policy %s overriden by force: %s" % (name, user["name"]) if reason: msg += ": %s" % reason @@ -12585,7 +12584,7 @@ class RootExports(object): values={'perm_id': perm_id}) update.set(description=description) update.execute() - if perm['name'] in koji.auth.get_user_perms(user_id): + if perm['name'] in get_user_perms(user_id): raise koji.GenericError('user %s already has permission: %s' % (userinfo, perm['name'])) insert = InsertProcessor('user_perms') @@ -12599,7 +12598,7 @@ class RootExports(object): user_id = get_user(userinfo, strict=True)['id'] perm = lookup_perm(permission, strict=True) perm_id = perm['id'] - if perm['name'] not in koji.auth.get_user_perms(user_id): + if perm['name'] not in get_user_perms(user_id): raise koji.GenericError('user %s does not have permission: %s' % (userinfo, perm['name'])) update = UpdateProcessor('user_perms', values=locals(), @@ -13299,7 +13298,7 @@ class RootExports(object): - userID: User ID or username. If no userID provided, current login user's permissions will be listed.""" user_info = get_user(userID, strict=True) - return koji.auth.get_user_perms(user_info['id']) + return get_user_perms(user_info['id']) def getAllPerms(self): """Get a list of all permissions in the system. Returns a list of maps. Each diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py index 30ca70c..5cae8b7 100644 --- a/kojihub/kojixmlrpc.py +++ b/kojihub/kojixmlrpc.py @@ -31,8 +31,6 @@ import traceback import re import koji -import koji.auth -import koji.db import koji.plugin import koji.policy import koji.util @@ -41,6 +39,8 @@ from koji.context import context # import xmlrpclib functions from koji to use tweaked Marshaller from koji.server import ServerError, BadRequest, RequestTimeout from koji.xmlrpcplus import ExtendedMarshaller, Fault, dumps, getparser +from kojihub import auth +from kojihub import db class Marshaller(ExtendedMarshaller): @@ -295,7 +295,7 @@ class ModXMLRPCRequestHandler(object): if not hasattr(context, "session"): # we may be called again by one of our meta-calls (like multiCall) # so we should only create a session if one does not already exist - context.session = koji.auth.Session() + context.session = auth.Session() try: context.session.validate() except koji.AuthLockError: @@ -343,7 +343,7 @@ class ModXMLRPCRequestHandler(object): results and errors, and return those as a list.""" results = [] for call in calls: - savepoint = koji.db.Savepoint('multiCall_loop') + savepoint = db.Savepoint('multiCall_loop') try: result = self._dispatch(call['methodName'], call['params']) except Fault as fault: @@ -729,13 +729,13 @@ def server_setup(environ): registry = get_registry(opts, plugins) policy = get_policy(opts, plugins) if opts.get('DBConnectionString'): - koji.db.provideDBopts(dsn=opts['DBConnectionString']) + db.provideDBopts(dsn=opts['DBConnectionString']) else: - koji.db.provideDBopts(database=opts["DBName"], - user=opts["DBUser"], - password=opts.get("DBPass", None), - host=opts.get("DBHost", None), - port=opts.get("DBPort", None)) + db.provideDBopts(database=opts["DBName"], + user=opts["DBUser"], + password=opts.get("DBPass", None), + host=opts.get("DBHost", None), + port=opts.get("DBPort", None)) except Exception: tb_str = ''.join(traceback.format_exception(*sys.exc_info())) logger.error(tb_str) @@ -785,7 +785,7 @@ def application(environ, start_response): context.environ = environ context.policy = policy try: - context.cnx = koji.db.connect() + context.cnx = db.connect() except Exception: return offline_reply(start_response, msg="database outage") h = ModXMLRPCRequestHandler(registry) @@ -844,13 +844,13 @@ def get_registry(opts, plugins): hostFunctions = kojihub.HostExports() registry.register_instance(functions) registry.register_module(hostFunctions, "host") - registry.register_function(koji.auth.login) - registry.register_function(koji.auth.sslLogin) - registry.register_function(koji.auth.logout) - registry.register_function(koji.auth.subsession) - registry.register_function(koji.auth.logoutChild) - registry.register_function(koji.auth.exclusiveSession) - registry.register_function(koji.auth.sharedSession) + registry.register_function(auth.login) + registry.register_function(auth.sslLogin) + registry.register_function(auth.logout) + registry.register_function(auth.subsession) + registry.register_function(auth.logoutChild) + registry.register_function(auth.exclusiveSession) + registry.register_function(auth.sharedSession) for name in opts.get('Plugins', '').split(): plugin = plugins.get(name) if not plugin: diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index 5e4b09f..3cb125e 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -18,7 +18,7 @@ import koji from koji.context import context from koji.plugin import callback, convert_datetime, ignore_error from kojihub import get_build_type -from koji.db import QueryProcessor, InsertProcessor, DeleteProcessor +from kojihub.db import QueryProcessor, InsertProcessor, DeleteProcessor CONFIG_FILE = '/etc/koji-hub/plugins/protonmsg.conf' CONFIG = None diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 9dce43c..f86f9fa 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later import koji -from koji.db import QueryProcessor, nextval from koji.context import context from koji.plugin import callback, export import koji.policy @@ -21,6 +20,7 @@ from kojihub import ( policy_get_user, readInheritanceData, ) +from kojihub.db import QueryProcessor, nextval CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" diff --git a/tests/test_hub/test_add_archivetype.py b/tests/test_hub/test_add_archivetype.py index 1434224..4e8e095 100644 --- a/tests/test_hub/test_add_archivetype.py +++ b/tests/test_hub/test_add_archivetype.py @@ -3,7 +3,6 @@ import unittest import mock import koji -import koji.db import kojihub import kojihub.kojihub diff --git a/tests/test_hub/test_add_host.py b/tests/test_hub/test_add_host.py index b4dfe1a..55b9238 100644 --- a/tests/test_hub/test_add_host.py +++ b/tests/test_hub/test_add_host.py @@ -42,7 +42,7 @@ class TestAddHost(unittest.TestCase): side_effect=self.getQuery).start() self.queries = [] self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context_db.session.assertLogin = mock.MagicMock() diff --git a/tests/test_hub/test_add_host_to_channel.py b/tests/test_hub/test_add_host_to_channel.py index 2ea9d49..8bb569c 100644 --- a/tests/test_hub/test_add_host_to_channel.py +++ b/tests/test_hub/test_add_host_to_channel.py @@ -20,7 +20,7 @@ class TestAddHostToChannel(unittest.TestCase): side_effect=self.getInsert).start() self.inserts = [] self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context_db.session.assertLogin = mock.MagicMock() diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 77a5698..e13cee3 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -18,7 +18,7 @@ class TestCGImporter(unittest.TestCase): if not os.path.exists(self.TMP_PATH): os.mkdir(self.TMP_PATH) self.path_work = mock.patch('koji.pathinfo.work').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.context = mock.patch('kojihub.kojihub.context').start() self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.get_user = mock.patch('kojihub.kojihub.get_user').start() @@ -271,7 +271,7 @@ class TestCGReservation(unittest.TestCase): self.inserts = [] self.updates = [] - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.context_db.session.user_id = 123456 self.mock_cursor = mock.MagicMock() self.context_db.cnx.cursor.return_value = self.mock_cursor diff --git a/tests/test_hub/test_complete_image_build.py b/tests/test_hub/test_complete_image_build.py index c59da58..ee828e1 100644 --- a/tests/test_hub/test_complete_image_build.py +++ b/tests/test_hub/test_complete_image_build.py @@ -52,7 +52,7 @@ class TestCompleteImageBuild(unittest.TestCase): self.pathinfo = koji.PathInfo(self.tempdir) mock.patch('koji.pathinfo', new=self.pathinfo).start() self.hostcalls = kojihub.HostExports() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() mock.patch('kojihub.kojihub.Host').start() self.Task = mock.patch('kojihub.kojihub.Task').start() self.Task.return_value.assertHost = mock.MagicMock() diff --git a/tests/test_hub/test_complete_maven_build.py b/tests/test_hub/test_complete_maven_build.py index 5763c08..17f4972 100644 --- a/tests/test_hub/test_complete_maven_build.py +++ b/tests/test_hub/test_complete_maven_build.py @@ -22,7 +22,7 @@ class TestCompleteMavenBuild(unittest.TestCase): mock.patch('koji.pathinfo', new=self.pathinfo).start() self.hostcalls = kojihub.HostExports() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.context.opts = {'EnableMaven': True} mock.patch('kojihub.kojihub.Host').start() self.Task = mock.patch('kojihub.kojihub.Task').start() @@ -34,8 +34,8 @@ class TestCompleteMavenBuild(unittest.TestCase): mock.patch.object(kojihub.BuildRoot, 'load', new=self.my_buildroot_load).start() mock.patch('kojihub.kojihub.import_archive_internal', new=self.my_import_archive_internal).start() - mock.patch('koji.db._dml').start() - mock.patch('koji.db._fetchSingle').start() + mock.patch('kojihub.db._dml').start() + mock.patch('kojihub.db._fetchSingle').start() mock.patch('kojihub.kojihub.build_notification').start() mock.patch('kojihub.kojihub.assert_policy').start() mock.patch('kojihub.kojihub.check_volume_policy', diff --git a/tests/test_hub/test_create_maven_build.py b/tests/test_hub/test_create_maven_build.py index 9ed1afe..8f13d43 100644 --- a/tests/test_hub/test_create_maven_build.py +++ b/tests/test_hub/test_create_maven_build.py @@ -15,7 +15,7 @@ class TestCreateMavenBuild(unittest.TestCase): self.exports = kojihub.RootExports() self.session = mock.MagicMock() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.context.session.assertPerm = mock.MagicMock() self.InsertProcessor = mock.patch('kojihub.kojihub.InsertProcessor', side_effect=self.getInsert).start() diff --git a/tests/test_hub/test_create_tag.py b/tests/test_hub/test_create_tag.py index 5f71483..16c7853 100644 --- a/tests/test_hub/test_create_tag.py +++ b/tests/test_hub/test_create_tag.py @@ -28,7 +28,7 @@ class TestCreateTag(unittest.TestCase): self.verify_name_internal = mock.patch('kojihub.kojihub.verify_name_internal').start() self.writeInheritanceData = mock.patch('kojihub.kojihub._writeInheritanceData').start() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context.session.assertPerm = mock.MagicMock() diff --git a/tests/test_hub/test_delete_build.py b/tests/test_hub/test_delete_build.py index 5714a50..119a51c 100644 --- a/tests/test_hub/test_delete_build.py +++ b/tests/test_hub/test_delete_build.py @@ -42,7 +42,7 @@ class TestDeleteBuild(unittest.TestCase): self.UpdateProcessor = mock.patch('kojihub.kojihub.UpdateProcessor', side_effect=self.getUpdate).start() self.updates = [] - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.context_db.session.assertLogin = mock.MagicMock() self.context_db.event_id = 42 self.context_db.session.user_id = 24 diff --git a/tests/test_hub/test_delete_tag.py b/tests/test_hub/test_delete_tag.py index 3a43ba4..759c1e4 100644 --- a/tests/test_hub/test_delete_tag.py +++ b/tests/test_hub/test_delete_tag.py @@ -20,7 +20,7 @@ class TestDeleteTag(unittest.TestCase): self.updates = [] self.get_tag = mock.patch('kojihub.kojihub.get_tag').start() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context.session.assertPerm = mock.MagicMock() diff --git a/tests/test_hub/test_edit_host.py b/tests/test_hub/test_edit_host.py index 8263bed..75719e4 100644 --- a/tests/test_hub/test_edit_host.py +++ b/tests/test_hub/test_edit_host.py @@ -30,7 +30,7 @@ class TestEditHost(unittest.TestCase): side_effect=self.getUpdate).start() self.updates = [] self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context_db.session.assertLogin = mock.MagicMock() diff --git a/tests/test_hub/test_edit_tag.py b/tests/test_hub/test_edit_tag.py index 8170380..d49b973 100644 --- a/tests/test_hub/test_edit_tag.py +++ b/tests/test_hub/test_edit_tag.py @@ -43,7 +43,7 @@ class TestEditTag(unittest.TestCase): self.get_perm_id = mock.patch('kojihub.kojihub.get_perm_id').start() self.verify_name_internal = mock.patch('kojihub.kojihub.verify_name_internal').start() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context_db.session.assertLogin = mock.MagicMock() diff --git a/tests/test_hub/test_get_active_repos.py b/tests/test_hub/test_get_active_repos.py index 4d156aa..f1a0632 100644 --- a/tests/test_hub/test_get_active_repos.py +++ b/tests/test_hub/test_get_active_repos.py @@ -2,7 +2,6 @@ import mock import unittest import koji import kojihub -import koji.db QP = kojihub.QueryProcessor diff --git a/tests/test_hub/test_get_user_perms.py b/tests/test_hub/test_get_user_perms.py index 6a9acee..948f22a 100644 --- a/tests/test_hub/test_get_user_perms.py +++ b/tests/test_hub/test_get_user_perms.py @@ -7,7 +7,7 @@ import kojihub class TestGetUserPerms(unittest.TestCase): def setUp(self): self.get_user = mock.patch('kojihub.kojihub.get_user').start() - self.get_user_perms = mock.patch('koji.auth.get_user_perms').start() + self.get_user_perms = mock.patch('kojihub.kojihub.get_user_perms').start() def tearDown(self): mock.patch.stopall() diff --git a/tests/test_hub/test_grant_permissions.py b/tests/test_hub/test_grant_permissions.py index 4dfb039..07afb97 100644 --- a/tests/test_hub/test_grant_permissions.py +++ b/tests/test_hub/test_grant_permissions.py @@ -18,9 +18,10 @@ class TestGrantPermission(unittest.TestCase): self.lookup_perm = mock.patch('kojihub.kojihub.lookup_perm').start() self.insert_processor = mock.patch('kojihub.kojihub.InsertProcessor').start() self.update_processor = mock.patch('kojihub.kojihub.UpdateProcessor').start() - self.get_user_perms = mock.patch('koji.auth.get_user_perms').start() - self.exports = kojihub.RootExports() + self.get_user_perms = mock.patch('kojihub.kojihub.get_user_perms').start() self.context = mock.patch('kojihub.kojihub.context').start() + self.context_db = mock.patch('kojihub.db.context').start() + self.exports = kojihub.RootExports() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context.session.assertPerm = mock.MagicMock() diff --git a/tests/test_hub/test_group_operations.py b/tests/test_hub/test_group_operations.py index 393d627..42beb76 100644 --- a/tests/test_hub/test_group_operations.py +++ b/tests/test_hub/test_group_operations.py @@ -41,7 +41,7 @@ class TestGrouplist(unittest.TestCase): def setUp(self): self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.get_tag = mock.patch('kojihub.kojihub.get_tag').start() self.lookup_tag = mock.patch('kojihub.kojihub.lookup_tag').start() self.lookup_group = mock.patch('kojihub.kojihub.lookup_group').start() diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index 8d5e537..564075b 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -22,7 +22,7 @@ class TestImportBuild(unittest.TestCase): self.check_volume_policy = mock.patch('kojihub.kojihub.check_volume_policy').start() self.new_typed_build = mock.patch('kojihub.kojihub.new_typed_build').start() - self._dml = mock.patch('koji.db._dml').start() + self._dml = mock.patch('kojihub.db._dml').start() self.nextval = mock.patch('kojihub.kojihub.nextval').start() self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.add_rpm_sig = mock.patch('kojihub.kojihub.add_rpm_sig').start() @@ -31,7 +31,7 @@ class TestImportBuild(unittest.TestCase): self.import_rpm = mock.patch('kojihub.kojihub.import_rpm').start() self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor').start() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.new_package = mock.patch('kojihub.kojihub.new_package').start() self.get_rpm_header = mock.patch('koji.get_rpm_header').start() self.pathinfo_work = mock.patch('koji.pathinfo.work').start() diff --git a/tests/test_hub/test_import_image_internal.py b/tests/test_hub/test_import_image_internal.py index 604afe6..a12d31e 100644 --- a/tests/test_hub/test_import_image_internal.py +++ b/tests/test_hub/test_import_image_internal.py @@ -11,7 +11,7 @@ class TestImportImageInternal(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.Task = mock.patch('kojihub.kojihub.Task').start() self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.get_archive_type = mock.patch('kojihub.kojihub.get_archive_type').start() diff --git a/tests/test_hub/test_import_rpm.py b/tests/test_hub/test_import_rpm.py index 3366431..014bdbb 100644 --- a/tests/test_hub/test_import_rpm.py +++ b/tests/test_hub/test_import_rpm.py @@ -30,7 +30,7 @@ class TestImportRPM(unittest.TestCase): pass self.context = mock.patch('kojihub.kojihub.context').start() self.context.session.assertPerm = mock.MagicMock() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.cursor = mock.MagicMock() self.rpm_header_retval = { diff --git a/tests/test_hub/test_multicall.py b/tests/test_hub/test_multicall.py index 0904374..5a9881a 100644 --- a/tests/test_hub/test_multicall.py +++ b/tests/test_hub/test_multicall.py @@ -16,7 +16,7 @@ class DummyExports(object): class TestMulticall(unittest.TestCase): def test_multicall(self): - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() kojixmlrpc.kojihub = mock.MagicMock() kojixmlrpc.context.opts = mock.MagicMock() kojixmlrpc.context.session = mock.MagicMock() diff --git a/tests/test_hub/test_remove_host_from_channel.py b/tests/test_hub/test_remove_host_from_channel.py index 03bd6ed..7122c5d 100644 --- a/tests/test_hub/test_remove_host_from_channel.py +++ b/tests/test_hub/test_remove_host_from_channel.py @@ -20,7 +20,7 @@ class TestRemoveHostFromChannel(unittest.TestCase): side_effect=self.getUpdate).start() self.updates = [] self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context_db.session.assertLogin = mock.MagicMock() diff --git a/tests/test_hub/test_set_host_enabled.py b/tests/test_hub/test_set_host_enabled.py index fc54acc..82bdf89 100644 --- a/tests/test_hub/test_set_host_enabled.py +++ b/tests/test_hub/test_set_host_enabled.py @@ -29,7 +29,7 @@ class TestSetHostEnabled(unittest.TestCase): side_effect=self.getUpdate).start() self.updates = [] self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.get_host = mock.patch('kojihub.kojihub.get_host').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" diff --git a/tests/test_hub/test_tag_operations.py b/tests/test_hub/test_tag_operations.py index c428628..b77bdf0 100644 --- a/tests/test_hub/test_tag_operations.py +++ b/tests/test_hub/test_tag_operations.py @@ -50,7 +50,7 @@ class TestTagBuild(unittest.TestCase): self.check_tag_access = mock.patch('kojihub.kojihub.check_tag_access').start() self.writeInheritanceData = mock.patch('kojihub.kojihub.writeInheritanceData').start() self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context.session.assertPerm = mock.MagicMock() diff --git a/tests/test_hub/test_user_groups.py b/tests/test_hub/test_user_groups.py index 6de8f1b..98e9b5c 100644 --- a/tests/test_hub/test_user_groups.py +++ b/tests/test_hub/test_user_groups.py @@ -32,7 +32,7 @@ class TestGrouplist(unittest.TestCase): def setUp(self): self.context = mock.patch('kojihub.kojihub.context').start() - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() self.get_user = mock.patch('kojihub.kojihub.get_user').start() self.verify_name_internal = mock.patch('kojihub.kojihub.verify_name_internal').start() # It seems MagicMock will not automatically handle attributes that diff --git a/tests/test_lib/test_auth.py b/tests/test_lib/test_auth.py index cb688e5..8b8f365 100644 --- a/tests/test_lib/test_auth.py +++ b/tests/test_lib/test_auth.py @@ -5,12 +5,11 @@ import mock import unittest import koji -import koji.auth -import koji.db import datetime +import kojihub.auth -UP = koji.auth.UpdateProcessor -QP = koji.auth.QueryProcessor +UP = kojihub.auth.UpdateProcessor +QP = kojihub.auth.QueryProcessor class TestAuthSession(unittest.TestCase): @@ -29,42 +28,41 @@ class TestAuthSession(unittest.TestCase): return query def setUp(self): - self.context = mock.patch('kojihub.context').start() - self.UpdateProcessor = mock.patch('koji.auth.UpdateProcessor', + self.context = mock.patch('kojihub.auth.context').start() + kojihub.db.context = self.context + self.UpdateProcessor = mock.patch('kojihub.auth.UpdateProcessor', side_effect=self.getUpdate).start() self.updates = [] self.query_execute = mock.MagicMock() self.query_executeOne = mock.MagicMock() self.query_singleValue = mock.MagicMock() - self.QueryProcessor = mock.patch('koji.auth.QueryProcessor', + self.QueryProcessor = mock.patch('kojihub.auth.QueryProcessor', side_effect=self.getQuery).start() self.queries = [] # It seems MagicMock will not automatically handle attributes that # start with "assert" self.context.session.assertLogin = mock.MagicMock() - @mock.patch('koji.auth.context') - def test_instance(self, context): - """Simple auth.Session instance""" - context.opts = { + def test_instance(self): + """Simple kojihub.auth.Session instance""" + self.context.opts = { 'CheckClientIP': True, 'DisableURLSessions': False, } with self.assertRaises(koji.GenericError) as cm: - koji.auth.Session() + kojihub.auth.Session() # no args in request/environment self.assertEqual(cm.exception.args[0], "'session-id' not specified in session args") - @mock.patch('koji.auth.context') - def get_session_old(self, context): + def get_session_old(self): """auth.Session instance""" # base session from test_basic_instance - # url-based auth - will be dropped in 1.34 - context.opts = { + # url-based kojihub.auth - will be dropped in 1.34 + self.context.opts = { 'CheckClientIP': True, 'DisableURLSessions': False, } - context.environ = { + self.context.environ = { 'QUERY_STRING': 'session-id=123&session-key=xyz&callnum=345', 'REMOTE_ADDR': 'remote-addr', } @@ -80,18 +78,17 @@ class TestAuthSession(unittest.TestCase): 'user_id': 1}, {'name': 'kojiadmin', 'status': 0, 'usertype': 0}] self.query_singleValue.return_value = 123 - s = koji.auth.Session() - return s, context + s = kojihub.auth.Session() + return s, self.context - @mock.patch('koji.auth.context') - def get_session(self, context): + def get_session(self): # base session from test_basic_instance # header-based auth - context.opts = { + self.context.opts = { 'CheckClientIP': True, 'DisableURLSessions': True, } - context.environ = { + self.context.environ = { 'HTTP_KOJI_SESSION_ID': '123', 'HTTP_KOJI_SESSION_KEY': 'xyz', 'HTTP_KOJI_CALLNUM': '345', @@ -109,8 +106,8 @@ class TestAuthSession(unittest.TestCase): 'user_id': 1}, {'name': 'kojiadmin', 'status': 0, 'usertype': 0}] self.query_singleValue.return_value = 123 - s = koji.auth.Session() - return s, context + s = kojihub.auth.Session() + return s, self.context def test_session_old(self): self.get_session_old() @@ -220,7 +217,7 @@ class TestAuthSession(unittest.TestCase): with self.assertRaises(AttributeError): s.non_existing_attribute - @mock.patch('koji.auth.context') + @mock.patch('auth.context') def test_str(self, context): """auth.Session string representation""" s, cntext = self.get_session() @@ -232,11 +229,10 @@ class TestAuthSession(unittest.TestCase): s.logged_in = True self.assertNotEqual(str(s), 'session: not logged in') - @mock.patch('koji.auth.context') - def test_validate(self, context): + def test_validate(self): """Session.validate""" s, cntext = self.get_session() - context.cnx = cntext.cnx + self.context.cnx = cntext.cnx s.lockerror = True with self.assertRaises(koji.AuthLockError): @@ -245,8 +241,7 @@ class TestAuthSession(unittest.TestCase): s.lockerror = False self.assertTrue(s.validate()) - @mock.patch('koji.auth.context') - def test_makeShared(self, context): + def test_makeShared(self): """Session.makeShared""" s, _ = self.get_session() s.makeShared() @@ -264,25 +259,24 @@ class TestAuthSession(unittest.TestCase): # all queries are tested in test_basic_instance @mock.patch('socket.gethostbyname') - @mock.patch('koji.auth.context') - def test_get_remote_ip(self, context, gethostbyname): + def test_get_remote_ip(self, gethostbyname): """Session.get_remote_ip""" - s, cntext = self.get_session() + s, _ = self.get_session() - context.opts = {'CheckClientIP': False} + self.context.opts = {'CheckClientIP': False} self.assertEqual(s.get_remote_ip(), '-') - context.opts = {'CheckClientIP': True} + self.context.opts = {'CheckClientIP': True} self.assertEqual(s.get_remote_ip(override='xoverride'), 'xoverride') - context.environ = {'REMOTE_ADDR': '123.123.123.123'} + self.context.environ = {'REMOTE_ADDR': '123.123.123.123'} self.assertEqual(s.get_remote_ip(), '123.123.123.123') gethostbyname.return_value = 'ip' - context.environ = {'REMOTE_ADDR': '127.0.0.1'} + self.context.environ = {'REMOTE_ADDR': '127.0.0.1'} self.assertEqual(s.get_remote_ip(), 'ip') - @mock.patch('koji.auth.context') + @mock.patch('auth.context') def test_login(self, context): s, cntext = self.get_session() @@ -329,13 +323,12 @@ class TestAuthSession(unittest.TestCase): with self.assertRaises(koji.AuthError): s.login('user', 'password') - @mock.patch('koji.auth.context') - def test_checkKrbPrincipal(self, context): + def test_checkKrbPrincipal(self): s, cntext = self.get_session() self.assertIsNone(s.checkKrbPrincipal(None)) - context.opts = {'AllowedKrbRealms': '*'} + self.context.opts = {'AllowedKrbRealms': '*'} self.assertIsNone(s.checkKrbPrincipal('any')) - context.opts = {'AllowedKrbRealms': 'example.com'} + self.context.opts = {'AllowedKrbRealms': 'example.com'} with self.assertRaises(koji.AuthError) as cm: s.checkKrbPrincipal('any') self.assertEqual(cm.exception.args[0], @@ -350,8 +343,7 @@ class TestAuthSession(unittest.TestCase): "Kerberos principal's realm:" " bannedrealm is not allowed") self.assertIsNone(s.checkKrbPrincipal('user@example.com')) - context.opts = {'AllowedKrbRealms': 'example.com,example.net' - ' , example.org'} + self.context.opts = {'AllowedKrbRealms': 'example.com,example.net,example.org'} self.assertIsNone(s.checkKrbPrincipal('user@example.net')) def test_getUserIdFromKerberos(self): @@ -420,7 +412,7 @@ class TestAuthSession(unittest.TestCase): s.logout() self.assertEqual(cm.exception.args[0], 'Not logged in') - @mock.patch('koji.auth.context') + @mock.patch('auth.context') def test_logout_logged(self, context): s, cntext = self.get_session() s.logged_in = True @@ -446,7 +438,7 @@ class TestAuthSession(unittest.TestCase): s.logoutChild(111) self.assertEqual(cm.exception.args[0], 'Not logged in') - @mock.patch('koji.auth.context') + @mock.patch('auth.context') def test_logoutChild_logged(self, context): s, cntext = self.get_session() s.logged_in = True @@ -493,7 +485,7 @@ class TestAuthSession(unittest.TestCase): self.assertEqual(len(self.queries), 5) self.assertEqual(len(self.updates), 2) - @mock.patch('koji.auth.context') + @mock.patch('auth.context') def test_makeExclusive(self, context): s, cntext = self.get_session() s.master = None @@ -635,13 +627,13 @@ class TestAuthSession(unittest.TestCase): # functions outside Session object def test_get_user_data(self): - """koji.auth.get_user_data""" + """auth.get_user_data""" self.query_executeOne.return_value = None self.assertEqual(len(self.queries), 0) self.query_executeOne.return_value = {'name': 'name', 'status': 'status', 'usertype': 'usertype'} - koji.auth.get_user_data(1) + kojihub.auth.get_user_data(1) self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['users']) @@ -650,8 +642,8 @@ class TestAuthSession(unittest.TestCase): self.assertEqual(query.columns, ['name', 'status', 'usertype']) def test_get_user_groups(self): - """koji.auth.get_user_groups""" - koji.auth.get_user_groups(1) + """auth.get_user_groups""" + kojihub.auth.get_user_groups(1) self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['user_groups']) @@ -661,8 +653,8 @@ class TestAuthSession(unittest.TestCase): self.assertEqual(query.columns, ['group_id', 'name']) def test_get_user_perms(self): - """koji.auth.get_user_perms""" - koji.auth.get_user_perms(1) + """auth.get_user_perms""" + kojihub.auth.get_user_perms(1) self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['user_perms']) @@ -670,14 +662,13 @@ class TestAuthSession(unittest.TestCase): self.assertEqual(query.clauses, ['active = TRUE', 'user_id=%(user_id)s']) self.assertEqual(query.columns, ['name']) - @mock.patch('koji.auth.context') - def test_logout_logged_not_owner(self, context): - s, cntext = self.get_session() + def test_logout_logged_not_owner(self): + s, _ = self.get_session() s.logged_in = True # session_id without admin perms and not owner - context.session.hasPerm.return_value = False - context.session.user_id.return_value = 123 + self.context.session.hasPerm.return_value = False + self.context.session.user_id.return_value = 123 self.query_singleValue.return_value = None with self.assertRaises(koji.ActionNotAllowed) as ex: s.logout(session_id=1) diff --git a/tests/test_lib/test_insert_processor.py b/tests/test_lib/test_insert_processor.py index 257e694..2c63450 100644 --- a/tests/test_lib/test_insert_processor.py +++ b/tests/test_lib/test_insert_processor.py @@ -7,7 +7,7 @@ import kojihub class TestInsertProcessor(unittest.TestCase): def setUp(self): - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() def tearDown(self): mock.patch.stopall() @@ -92,7 +92,7 @@ class TestInsertProcessor(unittest.TestCase): class TestBulkInsertProcessor(unittest.TestCase): def setUp(self): - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() def tearDown(self): mock.patch.stopall() diff --git a/tests/test_lib/test_query_processor.py b/tests/test_lib/test_query_processor.py index d9697a3..ab469cc 100644 --- a/tests/test_lib/test_query_processor.py +++ b/tests/test_lib/test_query_processor.py @@ -29,7 +29,7 @@ class TestQueryProcessor(unittest.TestCase): ) self.original_chunksize = kojihub.QueryProcessor.iterchunksize kojihub.QueryProcessor.iterchunksize = 2 - self.context_db = mock.patch('koji.db.context').start() + self.context_db = mock.patch('kojihub.db.context').start() def tearDown(self): kojihub.QueryProcessor.iterchunksize = self.original_chunksize @@ -125,7 +125,7 @@ class TestQueryProcessor(unittest.TestCase): result = next(generator) self.assertEqual(result, {'something': 'value number 3'}) - @mock.patch('koji.db._multiRow') + @mock.patch('kojihub.db._multiRow') def test_execution_as_list_transform(self, multirow): multirow.return_value = [{'col1': 'result_1_col_1', 'col2': 'result_1_col_2'}, {'col1': 'result_2_col_1', 'col2': 'result_2_col_2'}] diff --git a/tests/test_lib/test_savepoint.py b/tests/test_lib/test_savepoint.py index 5f34727..87a2e2c 100644 --- a/tests/test_lib/test_savepoint.py +++ b/tests/test_lib/test_savepoint.py @@ -8,8 +8,8 @@ import kojihub class TestSavepoint(unittest.TestCase): def setUp(self): - self.dml = mock.patch('koji.db._dml').start() - self.context_db = mock.patch('koji.db.context').start() + self.dml = mock.patch('kojihub.db._dml').start() + self.context_db = mock.patch('kojihub.db.context').start() def tearDown(self): mock.patch.stopall() diff --git a/tests/test_lib/test_update_processor.py b/tests/test_lib/test_update_processor.py index 8f95da7..aadfeab 100644 --- a/tests/test_lib/test_update_processor.py +++ b/tests/test_lib/test_update_processor.py @@ -21,7 +21,7 @@ class TestUpdateProcessor(unittest.TestCase): expected = {'data.foo': 'bar'} self.assertEqual(actual, expected) - @mock.patch('koji.db.context') + @mock.patch('kojihub.db.context') def test_simple_execution_with_iterate(self, context_db): cursor = mock.MagicMock() context_db.cnx.cursor.return_value = cursor diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 0c54e85..cd62346 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -6,8 +6,8 @@ from optparse import OptionParser from koji.context import context import koji -import koji.db -from koji.db import DeleteProcessor, QueryProcessor, BulkInsertProcessor +import kojihub.db +from kojihub.db import DeleteProcessor, QueryProcessor, BulkInsertProcessor def clean_sessions(cursor, vacuum, test, age, absolute): @@ -223,17 +223,17 @@ if __name__ == "__main__": opts[name] = default if opts.get('DBConnectionString'): - koji.db.provideDBopts(dsn=opts['DBConnectionString']) + kojihub.db.provideDBopts(dsn=opts['DBConnectionString']) else: if opts['DBHost'] is None: opts['DBHost'] = opts['DBhost'] - koji.db.provideDBopts(database=opts["DBName"], - user=opts["DBUser"], - password=opts.get("DBPass", None), - host=opts.get("DBHost", None), - port=opts.get("DBPort", None)) + kojihub.db.provideDBopts(database=opts["DBName"], + user=opts["DBUser"], + password=opts.get("DBPass", None), + host=opts.get("DBHost", None), + port=opts.get("DBPort", None)) - context.cnx = koji.db.connect() + context.cnx = kojihub.db.connect() context.cnx.set_session(autocommit=True) cursor = context.cnx.cursor() From e6cd6ad913beb609d155b24355524850f02f3e51 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Feb 20 2023 09:57:04 +0000 Subject: [PATCH 2/2] unify relative imports --- diff --git a/kojihub/auth.py b/kojihub/auth.py index 9ddb7da..1547fd7 100644 --- a/kojihub/auth.py +++ b/kojihub/auth.py @@ -33,7 +33,7 @@ import koji from koji.context import context from koji.util import to_list -from kojihub.db import DeleteProcessor, InsertProcessor, QueryProcessor, UpdateProcessor, nextval +from .db import DeleteProcessor, InsertProcessor, QueryProcessor, UpdateProcessor, nextval # 1 - load session if provided diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py index 5cae8b7..3139071 100644 --- a/kojihub/kojixmlrpc.py +++ b/kojihub/kojixmlrpc.py @@ -39,8 +39,8 @@ from koji.context import context # import xmlrpclib functions from koji to use tweaked Marshaller from koji.server import ServerError, BadRequest, RequestTimeout from koji.xmlrpcplus import ExtendedMarshaller, Fault, dumps, getparser -from kojihub import auth -from kojihub import db +from . import auth +from . import db class Marshaller(ExtendedMarshaller): diff --git a/tests/test_hub/test_check_volume_policy.py b/tests/test_hub/test_check_volume_policy.py index 09e7984..191fa3b 100644 --- a/tests/test_hub/test_check_volume_policy.py +++ b/tests/test_hub/test_check_volume_policy.py @@ -29,7 +29,7 @@ class TestCheckVolumePolicy(unittest.TestCase): def tearDown(self): mock.patch.stopall() - @mock.patch('kojixmlrpc.kojihub', new=kojihub, create=True) + @mock.patch('kojihub.kojixmlrpc.kojihub', new=kojihub, create=True) def load_policy(self, policy): '''policy is the policy dict with text values''' plugin = FakePlugin() diff --git a/tests/test_hub/test_kojixmlrpc.py b/tests/test_hub/test_kojixmlrpc.py index f81e1af..b085300 100644 --- a/tests/test_hub/test_kojixmlrpc.py +++ b/tests/test_hub/test_kojixmlrpc.py @@ -1,6 +1,6 @@ import unittest -import kojixmlrpc +from kojihub import kojixmlrpc class TestHandler(unittest.TestCase): diff --git a/tests/test_lib/test_auth.py b/tests/test_lib/test_auth.py index 8b8f365..2fae67a 100644 --- a/tests/test_lib/test_auth.py +++ b/tests/test_lib/test_auth.py @@ -30,6 +30,7 @@ class TestAuthSession(unittest.TestCase): def setUp(self): self.context = mock.patch('kojihub.auth.context').start() kojihub.db.context = self.context + kojihub.auth.context = self.context self.UpdateProcessor = mock.patch('kojihub.auth.UpdateProcessor', side_effect=self.getUpdate).start() self.updates = [] @@ -217,11 +218,10 @@ class TestAuthSession(unittest.TestCase): with self.assertRaises(AttributeError): s.non_existing_attribute - @mock.patch('auth.context') - def test_str(self, context): + def test_str(self): """auth.Session string representation""" s, cntext = self.get_session() - context.cnx = cntext.cnx + self.context.cnx = cntext.cnx s.logged_in = False s.message = 'msg' @@ -276,9 +276,8 @@ class TestAuthSession(unittest.TestCase): self.context.environ = {'REMOTE_ADDR': '127.0.0.1'} self.assertEqual(s.get_remote_ip(), 'ip') - @mock.patch('auth.context') - def test_login(self, context): - s, cntext = self.get_session() + def test_login(self): + s, _ = self.get_session() # already logged in with self.assertRaises(koji.GenericError): @@ -412,9 +411,8 @@ class TestAuthSession(unittest.TestCase): s.logout() self.assertEqual(cm.exception.args[0], 'Not logged in') - @mock.patch('auth.context') - def test_logout_logged(self, context): - s, cntext = self.get_session() + def test_logout_logged(self): + s, _ = self.get_session() s.logged_in = True s.logout() @@ -438,9 +436,8 @@ class TestAuthSession(unittest.TestCase): s.logoutChild(111) self.assertEqual(cm.exception.args[0], 'Not logged in') - @mock.patch('auth.context') - def test_logoutChild_logged(self, context): - s, cntext = self.get_session() + def test_logoutChild_logged(self): + s, _ = self.get_session() s.logged_in = True s.logoutChild(111) @@ -485,9 +482,8 @@ class TestAuthSession(unittest.TestCase): self.assertEqual(len(self.queries), 5) self.assertEqual(len(self.updates), 2) - @mock.patch('auth.context') - def test_makeExclusive(self, context): - s, cntext = self.get_session() + def test_makeExclusive(self): + s, _ = self.get_session() s.master = None s.exclusive = False self.query_singleValue.return_value = 123 diff --git a/tox.ini b/tox.ini index b38a6af..e794003 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ commands_pre = [testenv:py3] setenv = {[testenv]setenv} - PYTHONPATH=kojihub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/.:www/lib + PYTHONPATH=.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/.:www/lib commands_pre = {[testenv]commands_pre} {envbindir}/coverage3 erase --rcfile .coveragerc3