From d2915898cd4eb050138ccde5a4086d8ae3e8d34c Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 20 2018 17:13:28 +0000 Subject: Add hooks to repoSpanner setups This adds a hook runner script, which needs to be uploaded by the admin with the admin CLI upload-repospanner-hooks script. This returns a hook ID that is inserted into the configuration, after which it is automatically added to new repositories on repoSpanner. To make sure that the hooks are created for all repos, an admin ensure-project-hooks cli task is also added to make sure that all projects (Also those not on repoSpanner) have their hooks installed. Signed-off-by: Patrick Uiterwijk --- diff --git a/.gitignore b/.gitignore index 2519925..d99a465 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ dev/docker/test_env # A possible symlink for the testsuite /repospanner +/repohookrunner diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index c19e9d5..a1a9c1d 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -226,6 +226,7 @@ REPOSPANNER_ADMIN_MIGRATION = False # Example entry: # 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', # 'repo_prefix': 'pagure/', +# 'hook': None, # 'ca': '', # 'admin_cert': {'cert': '', # 'key': ''}, diff --git a/pagure/cli/admin.py b/pagure/cli/admin.py index 088d190..2e394af 100644 --- a/pagure/cli/admin.py +++ b/pagure/cli/admin.py @@ -14,6 +14,8 @@ import argparse import datetime import logging import os +import requests +from string import Template import sys import arrow @@ -340,6 +342,37 @@ def _parser_block_user(subparser): local_parser.set_defaults(func=do_block_user) +def _parser_upload_repospanner_hooks(subparser): + """ Set up the CLI argument parser to upload repospanner hook. + + Args: + subparser: An argparse subparser + """ + local_parser = subparser.add_parser( + "upload-repospanner-hook", help="Upload repoSpanner hook script" + ) + local_parser.add_argument( + "region", help="repoSpanner region where to " "upload hook" + ) + local_parser.set_defaults(func=do_upload_repospanner_hooks) + + +def _parser_ensure_project_hooks(subparser): + """ Set up the CLI argument parser to ensure project hooks are setup + + Args: + subparser: An argparse subparser + """ + local_parser = subparser.add_parser( + "ensure-project-hooks", + help="Ensure all projects have their hooks setup", + ) + local_parser.add_argument( + "hook", help="repoSpanner hook ID to set", default=None + ) + local_parser.set_defaults(func=do_ensure_project_hooks) + + def parse_arguments(args=None): """ Set-up the argument parsing. """ parser = argparse.ArgumentParser( @@ -386,6 +419,12 @@ def parse_arguments(args=None): # block-user _parser_block_user(subparser) + # upload-repospanner-hooks + _parser_upload_repospanner_hooks(subparser) + + # ensure-project-hooks + _parser_ensure_project_hooks(subparser) + return parser.parse_args(args) @@ -867,6 +906,74 @@ def do_block_user(args): session.commit() +def do_upload_repospanner_hooks(args): + """ Upload hooks to repoSpanner + + Args: + args (argparse.Namespace): Parsed arguments + """ + regioninfo = pagure.config.config["REPOSPANNER_REGIONS"].get(args.region) + if not regioninfo: + raise ValueError( + "repoSpanner region %s not in config file" % args.region + ) + + env = { + "config": os.environ.get("PAGURE_CONFIG", "/etc/pagure/pagure.cfg"), + "pypath": os.environ.get("PYTHONPATH", "None"), + } + sourcefile = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "../hooks/files/repospannerhook", + ) + ) + with open(sourcefile, "r") as source: + template = source.read() + hookcontents = Template(template).substitute(env) + + resp = requests.post( + "%s/admin/hook/admin.git/upload" % regioninfo["url"], + data=hookcontents, + headers={"X-Object-Size": str(len(hookcontents))}, + verify=regioninfo["ca"], + cert=( + regioninfo["admin_cert"]["cert"], + regioninfo["admin_cert"]["key"], + ), + ) + resp.raise_for_status() + resp = resp.json() + _log.debug("Response json: %s", resp) + if not resp["Success"]: + raise Exception("Error in repoSpanner API call: %s" % resp["Error"]) + hook = resp["Info"] + if hook == regioninfo["hook"]: + print("Hook was up-to-date") + else: + print("Hook ID for region %s: %s" % (args.region, hook)) + return hook + + +def do_ensure_project_hooks(args): + """ Ensures that all projects have their hooks setup + + Args: + args (argparse.Namespace): Parsed arguments + """ + projects = [] + query = session.query(pagure.lib.model.Project).order_by( + pagure.lib.model.Project.id + ) + for project in query.all(): + print("Ensuring hooks for %s" % project.fullname) + projects.append(project.fullname) + pagure.lib.git.set_up_project_hooks( + project, project.repospanner_region, hook=args.hook + ) + return projects + + def main(): """ Start of the application. """ diff --git a/pagure/default_config.py b/pagure/default_config.py index 3a8d44f..64d2c7f 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -495,6 +495,7 @@ REPOSPANNER_ADMIN_MIGRATION = False # Example entry: # 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', # 'repo_prefix': 'pagure/', +# 'hook': None, # 'ca': '', # 'admin_cert': {'cert': '', # 'key': ''}, diff --git a/pagure/hooks/__init__.py b/pagure/hooks/__init__.py index d5467fc..9a17e80 100644 --- a/pagure/hooks/__init__.py +++ b/pagure/hooks/__init__.py @@ -141,15 +141,12 @@ class BaseHook(object): # as the hook script will be arranged by repo creation. return - repopaths = [ - project.repopath(repotype) for repotype in pagure.lib.REPOTYPES - ] - hook_files = os.path.join( os.path.dirname(os.path.realpath(__file__)), "files" ) - for repopath in repopaths: + for repotype in pagure.lib.REPOTYPES: + repopath = project.repopath(repotype) if repopath is None: continue @@ -413,23 +410,33 @@ def run_project_hooks( raise SystemExit(1) -def run_hook_file(hooktype): - """ Runs a specific hook by grabbing the changes and running functions. +def extract_changes(from_stdin): + """ Extracts a changes dict from either stdin or argv Args: - hooktype (string): The name of the hook to run: pre-receive, update - or post-receive + from_stdin (bool): Whether to use stdin. If false, uses argv """ changes = {} - if hooktype in ("pre-receive", "post-receive"): + if from_stdin: for line in sys.stdin: (oldrev, newrev, refname) = line.strip().split(" ", 2) changes[refname] = (oldrev, newrev) - elif hooktype == "update": + else: (refname, oldrev, newrev) = sys.argv[1:] changes[refname] = (oldrev, newrev) - else: + return changes + + +def run_hook_file(hooktype): + """ Runs a specific hook by grabbing the changes and running functions. + + Args: + hooktype (string): The name of the hook to run: pre-receive, update + or post-receive + """ + if hooktype not in ("pre-receive", "update", "post-receive"): raise ValueError("Hook type %s not valid" % hooktype) + changes = extract_changes(from_stdin=hooktype != "update") session = pagure.lib.create_session(pagure_config["DB_URL"]) if not session: diff --git a/pagure/hooks/files/repospannerhook b/pagure/hooks/files/repospannerhook new file mode 100755 index 0000000..2669cda --- /dev/null +++ b/pagure/hooks/files/repospannerhook @@ -0,0 +1,68 @@ +#!/bin/env python3 +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Patrick Uiterwijk + +""" + +import os +import sys + +# These fields get filled in by upload-repospanner-hooks +os.environ['PAGURE_CONFIG'] = '${config}' +PYPATH = "${pypath}" + +# Prepare code imports +if PYPATH: + sys.path.append(PYPATH) + +import pagure +import pagure.lib +from pagure.hooks import run_project_hooks, extract_changes +from pagure.config import config as pagure_config + + +# Get information from the environment +hooktype = os.environ.get('HOOKTYPE') + +is_internal = os.environ.get('extra_internal', False) == 'yes' +pushuser = os.environ['extra_username'] +repotype = os.environ['extra_repotype'] +project_name = os.environ['extra_project_name'] +project_user = os.environ.get('extra_project_user', None) +project_namespace = os.environ.get('extra_project_namespace', None) +pruid = os.environ.get('extra_pull_request_uid', None) + +changes = extract_changes(from_stdin=hooktype != 'update') + +session = pagure.lib.create_session(pagure_config["DB_URL"]) +if not session: + raise Exception("Unable to initialize db session") + +gitdir = os.path.abspath(os.environ["GIT_DIR"]) + +project = pagure.lib._get_project( + session, project_name, project_user, project_namespace +) + +pull_request = None +if pruid: + pull_request = pagure.lib.get_request_by_uid( + session, os.environ["pull_request_uid"] + ) + +run_project_hooks( + session, + pushuser, + project, + hooktype, + repotype, + gitdir, + changes, + is_internal, + pull_request +) diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 50b8c1d..aa9121c 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -41,6 +41,7 @@ from pagure.config import config as pagure_config from pagure.lib import model from pagure.lib.repo import PagureRepo from pagure.lib import tasks +import pagure.hooks # from pagure.hooks import run_project_hooks @@ -957,6 +958,18 @@ class TemporaryClone(object): regioninfo = pagure_config["REPOSPANNER_REGIONS"][ self._project.repospanner_region ] + + extra.update( + { + "username": username, + "repotype": self._repotype, + "project_name": self._project.name, + "project_user": self._project.user.username + if self._project.is_fork + else "", + "project_namespace": self._project.namespace or "", + } + ) opts = [ "-c", "http.sslcainfo=%s" % regioninfo["ca"], @@ -964,8 +977,6 @@ class TemporaryClone(object): "http.sslcert=%s" % regioninfo["push_cert"]["cert"], "-c", "http.sslkey=%s" % regioninfo["push_cert"]["key"], - "-c", - "http.extraHeader=X-Extra-Username: %s" % username, ] for extrakey in extra: val = extra[extrakey] @@ -978,6 +989,7 @@ class TemporaryClone(object): "Running a git push of %s to %s" % (pushref, self._project.fullname) ) + _log.debug("Opts: %s", opts) env = os.environ.copy() env["GL_USER"] = username env.update(extra) @@ -993,6 +1005,7 @@ class TemporaryClone(object): # this way, we can be sure to get the output logged remotes = [] for line in ex.output.decode("utf-8").split("\n"): + _log.info("Remote line: %s", line) if line.startswith("remote: "): _log.debug("Remote: %s" % line) remotes.append(line[len("remote: ") :].strip()) @@ -2148,6 +2161,56 @@ def delete_project_repos(project): ) +def set_up_project_hooks(project, region, hook=None): + """ Makes sure the git repositories for a project have their hooks setup. + + Args: + project (model.Project): Project to set up hooks for + region (string or None): repoSpanner region to set hooks up for + hook (string): The hook ID to set up in repoSpanner (tests only) + """ + if region is None: + # This repo is not on repoSpanner, create hooks locally + pagure.hooks.BaseHook.set_up(project) + else: + regioninfo = pagure_config["REPOSPANNER_REGIONS"].get(region) + if not regioninfo: + raise ValueError( + "Invalid repoSpanner region %s looked up" % region + ) + if not hook: + hook = regioninfo["hook"] + if not hook: + # No hooks to set up for this region + return + + for repotype in pagure.lib.REPOTYPES: + data = { + "Reponame": project._repospanner_repo_name(repotype, region), + "UpdateRequest": { + "hook-prereceive": hook, + "hook-update": hook, + "hook-postreceive": hook, + }, + } + resp = requests.post( + "%s/admin/editrepo" % regioninfo["url"], + json=data, + verify=regioninfo["ca"], + cert=( + regioninfo["admin_cert"]["cert"], + regioninfo["admin_cert"]["key"], + ), + ) + resp.raise_for_status() + resp = resp.json() + _log.debug("Response json: %s", resp) + if not resp["Success"]: + raise Exception( + "Error in repoSpanner API call: %s" % resp["Error"] + ) + + def _create_project_repo(project, region, templ, ignore_existing, repotype): """ Creates a single specific git repository on disk or repoSpanner @@ -2242,3 +2305,5 @@ def create_project_repos(project, region, templ, ignore_existing): for created in created_dirs: shutil.rmtree(created) raise + + set_up_project_hooks(project, region) diff --git a/tests/__init__.py b/tests/__init__.py index aeb0d2e..ba00f40 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -106,6 +106,7 @@ REPOSPANNER_ADMIN_MIGRATION = %(repospanner_admin_migration)s REPOSPANNER_REGIONS = { 'default': {'url': 'https://nodea.regiona.repospanner.local:%(repospanner_gitport)s', 'repo_prefix': 'pagure/', + 'hook': None, 'ca': '%(path)s/repospanner/pki/ca.crt', 'admin_cert': {'cert': '%(path)s/repospanner/pki/admin.crt', 'key': '%(path)s/repospanner/pki/admin.key'}, diff --git a/tests/test_pagure_lib_git_auth.py b/tests/test_pagure_lib_git_auth.py index d6e0b6a..3f67d8a 100644 --- a/tests/test_pagure_lib_git_auth.py +++ b/tests/test_pagure_lib_git_auth.py @@ -93,19 +93,13 @@ class PagureLibGitAuthtests(tests.Modeltests): tests.add_content_git_repo( os.path.join(self.path, 'repos', 'hooktest.git')) - output = self.app.get('/hooktest/edit/master/f/sources') - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - csrf_token = output_text.split( - 'name="csrf_token" type="hidden" value="')[1].split('">')[0] - data = { 'content': 'foo\n bar\n baz', 'commit_title': 'test commit', 'commit_message': 'Online commits from the gure.lib.get', 'email': 'bar@pingou.com', 'branch': 'master', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( @@ -138,12 +132,6 @@ class PagureLibGitAuthtests(tests.Modeltests): tests.add_content_git_repo( os.path.join(self.path, 'repos', 'hooktest.git')) - output = self.app.get('/hooktest/edit/master/f/sources') - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - csrf_token = output_text.split( - 'name="csrf_token" type="hidden" value="')[1].split('">')[0] - # Try editing master branch, should fail (only PRs allowed) data = { 'content': 'foo\n bar\n baz', @@ -151,7 +139,7 @@ class PagureLibGitAuthtests(tests.Modeltests): 'commit_message': 'Online commits from the gure.lib.get', 'email': 'bar@pingou.com', 'branch': 'master', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( '/hooktest/edit/master/f/sources', data=data, @@ -172,7 +160,7 @@ class PagureLibGitAuthtests(tests.Modeltests): 'commit_message': 'Online commits from the gure.lib.get', 'email': 'bar@pingou.com', 'branch': 'source', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( diff --git a/tests/test_pagure_repospanner.py b/tests/test_pagure_repospanner.py index 91af881..81ac68a 100644 --- a/tests/test_pagure_repospanner.py +++ b/tests/test_pagure_repospanner.py @@ -14,6 +14,7 @@ __requires__ = ['SQLAlchemy >= 0.8'] import pkg_resources import datetime +import munch import unittest import shutil import subprocess @@ -34,6 +35,7 @@ sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) import pagure.lib +import pagure.cli.admin import tests @@ -80,11 +82,20 @@ hooks: hostname: myhostname bind: ro_bind: - /usr: /usr + - - /usr + - /usr + - - %(codepath)s + - %(codepath)s + - - %(path)s + - %(path)s + - - %(crosspath)s + - %(crosspath)s symlink: - usr/lib64: /lib64 - usr/bin: /bin - runner: /usr/bin/repohookrunner + - - usr/lib64 + - /lib64 + - - usr/bin + - /bin + runner: %(hookrunner_bin)s user: 0 """ @@ -123,6 +134,16 @@ class PagureRepoSpannerTests(tests.Modeltests): if not self.repospanner_binary: raise unittest.SkipTest('repoSpanner not found') + hookrunbin = os.path.join(os.path.dirname(self.repospanner_binary), + 'repohookrunner') + if not os.path.exists(hookrunbin): + raise Exception('repoSpanner found, but repohookrunner not') + + codepath = os.path.normpath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../")) + # Only run the setUp() function if we are actually going ahead and run # this test. The reason being that otherwise, setUp will set up a # database, but because we "error out" from setUp, the tearDown() @@ -132,8 +153,11 @@ class PagureRepoSpannerTests(tests.Modeltests): # TODO: Find free ports configvals = { 'path': self.path, + 'crosspath': tests.tests_state["path"], 'gitport': 8443, 'rpcport': 8444, + 'codepath': codepath, + 'hookrunner_bin': hookrunbin, } os.mkdir(os.path.join(self.path, 'repospanner')) @@ -216,7 +240,8 @@ class PagureRepoSpannerTests(tests.Modeltests): class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): config_values = { - 'repospanner_new_repo': "'default'" + 'repospanner_new_repo': "'default'", + 'authbackend': 'test_auth', } @patch('pagure.ui.app.admin_session_timedout') @@ -232,14 +257,11 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): self.assertIn( 'Create new Project', output_text) - csrf_token = output_text.split( - 'name="csrf_token" type="hidden" value="')[1].split('">')[0] - data = { 'name': 'project-1', 'description': 'Project #1', 'create_readme': 'y', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post('/new/', data=data, follow_redirects=True) @@ -258,9 +280,8 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): output.get_data(as_text=True)) with tests.user_set(self.app.application, tests.FakeUser(username='pingou')): - csrf_token = self.get_csrf() data = { - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( @@ -284,6 +305,113 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): repodirlist = os.listdir(os.path.join(self.path, 'repos')) self.assertEqual(repodirlist, ['pseudo']) + @patch('pagure.ui.app.admin_session_timedout') + def test_hooks(self, ast): + """ Test hook setting and running works. """ + ast.return_value = False + pagure.cli.admin.session = self.session + + # Upload the hook script to repoSpanner + args = munch.Munch({'region': 'default'}) + hookid = pagure.cli.admin.do_upload_repospanner_hooks(args) + + user = tests.FakeUser(username='foo') + with tests.user_set(self.app.application, user): + data = { + 'name': 'project-1', + 'description': 'Project #1', + 'create_readme': 'y', + 'csrf_token': self.get_csrf(), + } + + output = self.app.post('/new/', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + '
\nProject #1', + output_text) + self.assertIn( + 'Overview - project-1 - Pagure', output_text) + self.assertIn('Added the README', output_text) + + output = self.app.get('/project-1/settings') + self.assertIn( + 'This repository is on repoSpanner region default', + output.get_data(as_text=True)) + + # Check file before the commit: + output = self.app.get('/project-1/raw/master/f/README.md') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertEqual(output_text, '# project-1\n\nProject #1') + + # Set the hook + args = munch.Munch({'hook': hookid}) + projects = pagure.cli.admin.do_ensure_project_hooks(args) + self.assertEqual(["project-1"], projects) + + with tests.user_set(self.app.application, user): + # Set editing Denied + self.set_auth_status(False) + + # Try to make an edit in the repo + data = { + 'content': 'foo\n bar\n baz', + 'commit_title': 'test commit', + 'commit_message': 'Online commit', + 'email': 'foo@bar.com', + 'branch': 'master', + 'csrf_token': self.get_csrf(), + } + + output = self.app.post( + '/project-1/edit/master/f/README.md', data=data, + follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + "Remote hook declined the push: ", + output_text + ) + self.assertIn( + "Denied push for ref 'refs/heads/master' for user 'foo'\n" + "All changes have been rejected", + output_text + ) + + # Check file after the commit: + output = self.app.get('/project-1/raw/master/f/README.md') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertEqual(output_text, '# project-1\n\nProject #1') + + # Set editing Allowed + self.set_auth_status(True) + + # Try to make an edit in the repo + data = { + 'content': 'foo\n bar\n baz', + 'commit_title': 'test commit', + 'commit_message': 'Online commit', + 'email': 'foo@bar.com', + 'branch': 'master', + 'csrf_token': self.get_csrf(), + } + + output = self.app.post( + '/project-1/edit/master/f/README.md', data=data, + follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Commits - project-1 - Pagure', output_text) + + # Check file after the commit: + output = self.app.get('/project-1/raw/master/f/README.md') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertEqual(output_text, 'foo\n bar\n baz') + if __name__ == '__main__': unittest.main(verbosity=2)