From fcd52e0d56f2c1c2e36838242a050a9ce81377d2 Mon Sep 17 00:00:00 2001 From: Slavek Kabrda Date: Mar 26 2018 15:15:16 +0000 Subject: Implement an optimed Gitolite3PythonAuth backend --- diff --git a/pagure/lib/git_auth.py b/pagure/lib/git_auth.py index 149f852..d6d3b2a 100644 --- a/pagure/lib/git_auth.py +++ b/pagure/lib/git_auth.py @@ -656,6 +656,286 @@ class Gitolite3Auth(Gitolite2Auth): return cmd +class Gitolite3PythonAuth(Gitolite3Auth): + """ Replaces ``gitolite compile && gitolite trigger POST_COMPILE`` calls + by implementing used subset of gitolite's functionality in Python + to achieve a major speedup with some limitations: + + * This implementation can only work with exactly the subset of gitolite + functionality that Pagure uses. + * Currently it doesn't respect ``GITOLITE_PRE_CONFIG`` and + ``GITOLITE_POST_CONFIG`` configuration settings. + * This implementation doesn't support: + + * specifying repos with templates/wildcards + * groups being members of other groups + * specifying access rights to specific branches + * anything that ``gitolite trigger POST_COMPILE`` does, other + than handling ssh keys uploaded by users + * probably most gitolite.rc non-default options + * maybe more: TODO + """ + + update_hook_content = """#!/usr/bin/perl +use strict; +use warnings; +use lib $ENV{GL_LIBDIR}; +use Gitolite::Hooks::Update; +# gitolite update hook +# ---------------------------------------------------------------------- +update(); # is not expected to return +exit 1; # so if it does, something is wrong +""" + + @classmethod + def _read_gitolite_config(cls, configfile): + """ Read given configfile (file with gitolite.conf syntax) + into a dict. + + Example result: + { + 'groups': + { + 'group1': set(['user1', 'user2']), + 'group2': set(['user2', 'user3']), + 'group3': set(['@group1', '@group2']) + }, + 'repos': + { + 'repo1': + { + 'R': set(['user1']), + 'RW': set(['@group1', 'user3']) + } + } + } + } + """ + config = {'groups': {}, 'repos': {}} + configlines = cls._get_current_config(configfile) + current_repo = None + + for line in configlines: + if line.startswith('@'): + group, members = line.split('=') + groupname = group[1:].strip() + config['groups'].setdefault(groupname, set()) + config['groups'][groupname].update(members.split()) + elif line.startswith('repo '): + reponame = line.split()[1] + config['repos'].setdefault(reponame, {}) + current_repo = reponame + elif line.startswith((' ', '\t')): + # note: this might also be in form " R master = user1 user2", + # but I think Pagure doesn't use that anywhere + access, who = line.strip().split('=') + access = access.strip() + config['repos'][current_repo].setdefault(access, set()) + config['repos'][current_repo][access].update(who.split()) + elif line.startswith('#') or not line.strip(): + continue + else: + msg = 'Unexpected line in gitolite.conf: "%s"' % line + raise pagure.exceptions.PagureException(msg) + + return config + + @classmethod + def _get_changed_repos_names(cls, old_config, new_config): + """ Compare old and new configs loaded by _read_gitolite_config + and return lists with names of created, modified and deleted repos. + + """ + old = old_config['repos'] + new = new_config['repos'] + + old_names = set(old.keys()) + new_names = set(new.keys()) + + created = list(new_names - old_names) + deleted = list(old_names - new_names) + modified = [] + + for k, v in old.items(): + if k in new and v != new[k]: + modified.append(k) + + return created, modified, deleted + + @classmethod + def _write_repo_files(cls, name, repo_config): + """ Write all files necessary for gitolite to operate correctly + with given repo. + """ + _log.info('Writing repo files for repo %s', name) + repodir = os.path.join(pagure_config['GIT_FOLDER'], name + '.git') + if not os.path.exists(repodir): + # pagure generates config for e.g. tickets even if they're disabled + return + glconf = os.path.join(repodir, 'gl-conf') + # repo_config is mapping like {'RW': set('user1', '@group1')}, + # we need to get {'user1': [[, 'RW', 'refs/.*']]} + # the number doesn't seem to matter much, it just has to be + # unique in this gl-conf + access_conf = {} + counter = 1000 + for perm, users in repo_config.items(): + for user in users: + access_conf.setdefault(user, []) + access_conf[user].append([counter, perm, 'refs/.*']) + counter += 1 + perms = [] + for user, user_perms_list in sorted(access_conf.items(), + key=lambda x: x[0]): + perms.append(" '%s' => %s" % (user, str(user_perms_list))) + with open(glconf, 'w') as f: + f.write("%one_repo = (\n") + f.write(" '%s' => {\n" % name) + f.write(",\n".join(perms)) + f.write("\n }\n") + f.write(");\n") + + update_hook = os.path.join(repodir, 'hooks', 'update') + with open(update_hook, 'w') as f: + f.write(cls.update_hook_content) + + # TODO: do we need git-daemon-export-ok? seems like not + + @classmethod + def _write_gitolite_compiled(cls, new_config): + """ Write gitolite.conf-compiled.pm from in-memory structure + as returned by _read_gitolite_config. + """ + _log.info('Writing compiled gitolite config') + compiled_path = pagure_config['GITOLITE_CONFIG'] + '-compiled.pm' + # new_config['groups'] is mapping of group name to list of members, + # but we need mapping of username to groupnames + user_groups = {} + for groupn, users in new_config['groups'].items(): + # theoretically, groups can contain other groups, but Pagure + # seems to not use this functionality, so we don't handle it + for usern in users: + user_groups.setdefault(usern, []) + user_groups[usern].append('@' + groupn) + groups = [] + for user, group_names in sorted(user_groups.items(), + key=lambda x: x[0]): + groups.append(" '%s' => %s" % (user, str(sorted(group_names)))) + split_conf = [] + for repo in sorted(new_config['repos'].keys()): + split_conf.append(" '%s' => 1" % repo) + with open(compiled_path, 'w') as f: + f.write("$data_version = '3.2';\n") + f.write("%repos = ();\n") + f.write("%groups = (\n") + f.write(",\n".join(groups)) + f.write("\n);\n") + f.write("%split_conf = (\n") + f.write(",\n".join(split_conf)) + f.write("\n);\n") + + @classmethod + def _handle_ssh_keys(cls): + """ Invoke the part of gitolite's "trigger POST_COMPILE" functionality + that handles ssh keys. + """ + _log.info('Handling user ssh keys') + cmd = '%(home)s GL_BINDIR=`%(home)s gitolite query-rc GL_BINDIR` ' + cmd += 'GL_LIBDIR=`%(home)s gitolite query-rc GL_LIBDIR` ' + cmd += '/usr/share/gitolite3/triggers/post-compile/ssh-authkeys' + proc = subprocess.Popen( + cmd % {'home': 'HOME=/var/pagure'}, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=pagure_config['GITOLITE_HOME'] + ) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + error_msg = ( + 'The command "{0}" failed with' + '\n\n out: "{1}\n\n err:"{2}"' + .format(cmd, stdout, stderr)) + raise pagure.exceptions.PagureException(error_msg) + + @classmethod + def generate_acls(cls, project, group=None): + """ Generate the gitolite configuration file for all repos + + :arg project: the project to update in the gitolite configuration + file. It can also be ``-1`` or ``None``, which is used + by the Gitolite3Auth, but doesn't make any difference + for this implementation of ``generate_acls``. + :type project: None, int or pagure.lib.model.Project + :kwarg group: the group to refresh the members of + :type group: None or pagure.lib.model.PagureGroup + + """ + _log.info('Refresh gitolite configuration') + + # this method could definitely use some optimization + old_config = cls._read_gitolite_config( + pagure_config['GITOLITE_CONFIG']) + + if project is not None or group is not None: + cls.write_gitolite_acls( + pagure.SESSION, + project=project, + configfile=pagure_config['GITOLITE_CONFIG'], + preconf=None, + postconf=None, + group=group, + ) + + new_config = cls._read_gitolite_config( + pagure_config['GITOLITE_CONFIG']) + + created, modified, deleted = cls._get_changed_repos_names( + old_config, new_config) + + _log.info('Changed repos: %s created, %s modified, %s deleted', + created, modified, deleted) + # we don't really care about deleted repos - these will get rm -rf'ed + # and since they're not in new_config, they won't end up in pm-compiled + for repo in created + modified: + cls._write_repo_files(repo, new_config['repos'][repo]) + + cls._write_gitolite_compiled(new_config) + + if project is None and group is None: + cls._handle_ssh_keys() + + @classmethod + def remove_acls(cls, session, project): + """ Remove a project from the configuration file for gitolite. + + :arg session: the session with which to connect to the database + :arg project: the project to remove from the gitolite configuration + file. + :type project: pagure.lib.model.Project + + """ + # this method could definitely use some optimization as well :) + cfile = pagure_config['GITOLITE_CONFIG'] + config = cls._get_current_config(cfile) + config = cls._clean_current_config(config, project) + _log.info('Writing configuration file after deleting project') + with open(cfile, 'w') as stream: + prev = None + for row in config: + if prev is None: + prev = row + if prev == row == '': + continue + stream.write(row + '\n') + prev = row + + stream.write('# end of body\n') + # we just removed the old project + # let generate_acls regenerate everything + cls.generate_acls(project=None) + + class GitAuthTestHelper(GitAuthHelper): """ Simple test auth module to check the auth customization system. """