From a21c5608075b27e02ffc1523a7246da502cff13d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 22 2018 16:14:42 +0000 Subject: [PATCH 1/7] Fix import in the pagure_hook file Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/hooks/files/pagure_hook.py b/pagure/hooks/files/pagure_hook.py index d6c8f99..14ba072 100755 --- a/pagure/hooks/files/pagure_hook.py +++ b/pagure/hooks/files/pagure_hook.py @@ -20,7 +20,7 @@ if 'PAGURE_CONFIG' not in os.environ \ os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' -import pagure # noqa: E402 +import pagure.config # noqa: E402 import pagure.exceptions # noqa: E402 import pagure.lib.link # noqa: E402 From 86e98dbcdbbcb8392a7782cf1f00c2188f08e2c9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 22 2018 16:14:42 +0000 Subject: [PATCH 2/7] Add a way to return both stdout and stderr when calling out sub-command Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index aa54fcf..c88fb64 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -1002,8 +1002,29 @@ def _update_file_in_git( return os.path.join('files', filename) -def read_output(cmd, abspath, input=None, keepends=False, **kw): - """ Read the output from the given command to run """ +def read_output(cmd, abspath, input=None, keepends=False, error=False, **kw): + """ Read the output from the given command to run. + + cmd: + The command to run, this is a list with each space separated into an + element of the list. + abspath: + The absolute path where the command should be ran. + input: + Whether the command should take input from stdin or not. + (Defaults to False) + keepends: + Whether to strip the newline characters at the end of the standard + output or not. + error: + Whether to return both the standard output and the standard error, + or just the standard output. + (Defaults to False). + kw*: + Any other arguments to be passed onto the subprocess.Popen command, + such as env, shell, executable... + + """ if input: stdin = subprocess.PIPE else: @@ -1025,24 +1046,35 @@ def read_output(cmd, abspath, input=None, keepends=False, **kw): print(err) if not keepends: out = out.rstrip('\n\r') - return out + + if error: + return (out, err) + else: + return out -def read_git_output(args, abspath, input=None, keepends=False, **kw): +def read_git_output( + args, abspath, input=None, keepends=False, error=False, **kw): """Read the output of a Git command.""" return read_output( - ['git'] + args, abspath, input=input, keepends=keepends, **kw) + ['git'] + args, abspath, input=input, + keepends=keepends, error=error, **kw) -def read_git_lines(args, abspath, keepends=False, **kw): +def read_git_lines(args, abspath, keepends=False, error=False, **kw): """Return the lines output by Git command. Return as single lines, with newlines stripped off.""" - return read_git_output( - args, abspath, keepends=keepends, **kw - ).splitlines(keepends) + if error: + return read_git_output( + args, abspath, keepends=keepends, error=error, **kw + ) + else: + return read_git_output( + args, abspath, keepends=keepends, **kw + ).splitlines(keepends) def get_revs_between(oldrev, newrev, abspath, refname, forced=False): From a3c32f97f88a8af545e8c50aa9db1ec96f13d07f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 22 2018 16:14:42 +0000 Subject: [PATCH 3/7] Allow plugins to have readonly fields in their form Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/_formhelper.html b/pagure/templates/_formhelper.html index 1efad34..cf7fed6 100644 --- a/pagure/templates/_formhelper.html +++ b/pagure/templates/_formhelper.html @@ -71,10 +71,10 @@ {% endmacro %} -{% macro render_field_in_row(field, after="") %} +{% macro render_field_in_row(field, after="", readonly=False) %} {{ field.label }} - {{ field(class="form-control")|safe }} + {{ field(class="form-control", readonly=readonly)|safe }} {% if after %} {{ after }}{% endif %} {% if field.errors %} {% for error in field.errors %} diff --git a/pagure/templates/plugin.html b/pagure/templates/plugin.html index b13fe4f..fbb5dd5 100644 --- a/pagure/templates/plugin.html +++ b/pagure/templates/plugin.html @@ -22,7 +22,11 @@ {% for field in fields %} + {% if field.id in form_fields_readonly %} + {{ render_field_in_row(field, readonly=True) }} + {% else %} {{ render_field_in_row(field) }} + {% endif %} {% endfor %}
diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 0f4b5bf..685c7f0 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -108,6 +108,10 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): for field in plugin.form_fields: fields.append(getattr(form, field)) + form_fields_readonly = [] + if hasattr(plugin, 'form_fields_readonly'): + form_fields_readonly = plugin.form_fields_readonly + if form.validate_on_submit(): form.populate_obj(obj=dbobj) @@ -174,4 +178,6 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): username=username, plugin=plugin, form=form, - fields=fields) + fields=fields, + form_fields_readonly=form_fields_readonly, + ) From 893d4f96c724ac8f2c2d01a97c4f3283081eb11b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 23 2018 08:02:43 +0000 Subject: [PATCH 4/7] Add a mirroring hook to mirror git repo to other locations Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/default_config.py b/pagure/default_config.py index 49231f0..5304200 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -105,6 +105,7 @@ WEBHOOK_CELERY_QUEUE = 'pagure_webhook' LOGCOM_CELERY_QUEUE = 'pagure_logcom' LOADJSON_CELERY_QUEUE = 'pagure_loadjson' CI_CELERY_QUEUE = 'pagure_ci' +MIRRORING_QUEUE = 'pagure_mirror' # Number of items displayed per page ITEM_PER_PAGE = 48 @@ -126,6 +127,9 @@ REDIS_PORT = 6379 REDIS_DB = 0 EVENTSOURCE_PORT = 8080 +# Folder where to place the ssh keys for the mirroring feature +MIRROR_SSHKEYS_FOLDER = '/var/lib/pagure/sshkeys/' + # Folder containing to the git repos # Note that this must be exactly the same as GL_REPO_BASE in gitolite.rc GIT_FOLDER = os.path.join( diff --git a/pagure/hooks/files/mirror.py b/pagure/hooks/files/mirror.py new file mode 100755 index 0000000..769a3a5 --- /dev/null +++ b/pagure/hooks/files/mirror.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python + + +"""Pagure specific hook to mirror a repo to another location. +""" +from __future__ import unicode_literals, print_function + + +import logging +import os +import sys + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure.config # noqa: E402 +import pagure.exceptions # noqa: E402 +import pagure.lib # noqa: E402 +import pagure.lib.tasks_mirror # noqa: E402 +import pagure.ui.plugins # noqa: E402 + + +_log = logging.getLogger(__name__) +_config = pagure.config.config +abspath = os.path.abspath(os.environ['GIT_DIR']) + + +def main(args): + + repo = pagure.lib.git.get_repo_name(abspath) + username = pagure.lib.git.get_username(abspath) + namespace = pagure.lib.git.get_repo_namespace(abspath) + if _config.get('HOOK_DEBUG', False): + print('repo:', repo) + print('user:', username) + print('namespace:', namespace) + + session = pagure.lib.create_session(_config['DB_URL']) + project = pagure.lib._get_project( + session, repo, user=username, namespace=namespace) + + if not project: + print('Could not find a project corresponding to this git repo') + session.close() + return 1 + + pagure.lib.tasks_mirror.mirror_project.delay( + username=project.user.user if project.is_fork else None, + namespace=project.namespace, + name=project.name) + + session.close() + return 0 + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py new file mode 100644 index 0000000..b874738 --- /dev/null +++ b/pagure/hooks/mirror_hook.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2016-2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + + +import sqlalchemy as sa +import wtforms + +try: + from flask_wtf import FlaskForm +except ImportError: + from flask_wtf import Form as FlaskForm +from sqlalchemy.orm import relation +from sqlalchemy.orm import backref + +import pagure.lib.tasks_mirror +from pagure.hooks import BaseHook, RequiredIf +from pagure.lib.model import BASE, Project +from pagure.utils import get_repo_path, ssh_urlpattern + + +class MirrorTable(BASE): + """ Stores information about the mirroring hook deployed on a project. + + Table -- mirror_pagure + """ + + __tablename__ = 'hook_mirror' + + id = sa.Column(sa.Integer, primary_key=True) + project_id = sa.Column( + sa.Integer, + sa.ForeignKey( + 'projects.id', onupdate='CASCADE', ondelete='CASCADE'), + nullable=False, + unique=True, + index=True) + + active = sa.Column(sa.Boolean, nullable=False, default=False) + + public_key = sa.Column(sa.Text, nullable=True) + target = sa.Column(sa.Text, nullable=True) + last_log = sa.Column(sa.Text, nullable=True) + + project = relation( + 'Project', remote_side=[Project.id], + backref=backref( + 'mirror_hook', cascade="delete, delete-orphan", + single_parent=True, uselist=False) + ) + + +class CustomRegexp(wtforms.validators.Regexp): + + def __init__(self, *args, **kwargs): + self.optional = kwargs.get('optional') or False + if self.optional: + kwargs.pop('optional') + super(CustomRegexp, self).__init__(*args, **kwargs) + + def __call__(self, form, field): + if self.optional: + if field.data: + return super(CustomRegexp, self).__call__(form, field) + else: + return super(CustomRegexp, self).__call__(form, field) + + +class MirrorForm(FlaskForm): + ''' Form to configure the mirror hook. ''' + active = wtforms.BooleanField( + 'Active', + [wtforms.validators.Optional()] + ) + + target = wtforms.TextField( + 'Git repo to mirror to', + [ + RequiredIf('active'), + CustomRegexp(ssh_urlpattern, optional=True), + ] + ) + + public_key = wtforms.TextAreaField( + 'Public SSH key', + [wtforms.validators.Optional()] + ) + last_log = wtforms.TextAreaField( + 'Log of the last sync:', + [wtforms.validators.Optional()] + ) + + +DESCRIPTION = ''' +Pagure specific hook to mirror a repo hosted on pagure to another location. + +The first field below should contain the URL to be set in the git configuration +as the URL of the git repository to mirror to. +It's format is going to be something like: + + @: + +The public SSH key is being generated by pagure and will be available in this +page shortly after the activation of this hook. Just refresh the page until +it shows up. + +Finally the log of the last sync at the bottom is meant. +''' + + +class MirrorHook(BaseHook): + ''' Mirror hook. ''' + + name = 'Mirroring' + description = DESCRIPTION + form = MirrorForm + db_object = MirrorTable + backref = 'mirror_hook' + form_fields = ['active', 'target', 'public_key', 'last_log'] + form_fields_readonly = ['public_key', 'last_log'] + + @classmethod + def install(cls, project, dbobj): + ''' Method called to install the hook for a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + pagure.lib.tasks_mirror.setup_mirroring.delay( + username=project.user.user if project.is_fork else None, + namespace=project.namespace, + name=project.name) + + repopaths = [get_repo_path(project)] + cls.base_install(repopaths, dbobj, 'mirror', 'mirror.py') + + @classmethod + def remove(cls, project): + ''' Method called to remove the hook of a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + pagure.lib.tasks_mirror.teardown_mirroring.delay( + username=project.user.user if project.is_fork else None, + namespace=project.namespace, + name=project.name) + + repopaths = [get_repo_path(project)] + cls.base_remove(repopaths, 'mirror') diff --git a/pagure/lib/tasks_mirror.py b/pagure/lib/tasks_mirror.py new file mode 100644 index 0000000..f64d423 --- /dev/null +++ b/pagure/lib/tasks_mirror.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +import base64 +import logging +import os +import shutil +import stat +import struct +import tempfile + +import pygit2 +import six +import werkzeug + +from celery import Celery +from cryptography import utils +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +import pagure.lib +from pagure.config import config as pagure_config +from pagure.lib.tasks import pagure_task +from pagure.utils import ssh_urlpattern + +# logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1}) +_log = logging.getLogger(__name__) + + +if os.environ.get('PAGURE_BROKER_URL'): # pragma: no-cover + broker_url = os.environ['PAGURE_BROKER_URL'] +elif pagure_config.get('BROKER_URL'): + broker_url = pagure_config['BROKER_URL'] +else: + broker_url = 'redis://%s' % pagure_config['REDIS_HOST'] + +conn = Celery('tasks_mirror', broker=broker_url, backend=broker_url) +conn.conf.update(pagure_config['CELERY_CONFIG']) + + +# Code from: +# https://github.com/pyca/cryptography/blob/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/primitives/serialization.py#L161 +# Taken from upstream cryptography since the version we have is too old +# and doesn't have this code (yet) +def _ssh_write_string(data): + return struct.pack(">I", len(data)) + data + + +def _ssh_write_mpint(value): + data = utils.int_to_bytes(value) + if six.indexbytes(data, 0) & 0x80: + data = b"\x00" + data + return _ssh_write_string(data) + + +# Code from _openssh_public_key_bytes at: +# https://github.com/pyca/cryptography/tree/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/backends/openssl#L1616 +# Taken from upstream cryptography since the version we have is too old +# and doesn't have this code (yet) +def _serialize_public_ssh_key(key): + if isinstance(key, rsa.RSAPublicKey): + public_numbers = key.public_numbers() + return b"ssh-rsa " + base64.b64encode( + _ssh_write_string(b"ssh-rsa") + + _ssh_write_mpint(public_numbers.e) + + _ssh_write_mpint(public_numbers.n) + ) + else: + # Since we only write RSA keys, drop the other serializations + return + + +def _create_ssh_key(keyfile): + ''' Create the public and private ssh keys. + + The specified file name will be the private key and the public one will + be in a similar file name ending with a '.pub'. + + ''' + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + with os.fdopen(os.open( + keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb')\ + as stream: + stream.write(private_pem) + + public_key = private_key.public_key() + public_pem = _serialize_public_ssh_key(public_key) + if public_pem: + with open(keyfile + '.pub', 'wb') as stream: + stream.write(public_pem) + + +@conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True) +@pagure_task +def setup_mirroring(self, session, username, namespace, name): + ''' Setup the specified project for mirroring. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + plugin.db_object() + + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=username) + + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER'] + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder, mode=0o700) + else: + if os.path.islink(ssh_folder): + raise pagure.exceptions.PagureException( + 'SSH folder is a link') + folder_stat = os.stat(ssh_folder) + filemode = stat.S_IMODE(folder_stat.st_mode) + if filemode != int('0700', 8): + raise pagure.exceptions.PagureException( + 'SSH folder had invalid permissions') + if folder_stat.st_uid != os.getuid() \ + or folder_stat.st_gid != os.getgid(): + raise pagure.exceptions.PagureException( + 'SSH folder does not belong to the user or group running ' + 'this task') + + public_key_file = os.path.join(ssh_folder, '%s.pub' % public_key_name) + _log.info('Public key of interest: %s', public_key_file) + + if os.path.exists(public_key_file): + raise pagure.exceptions.PagureException('SSH key already exists') + + _log.info('Creating public key') + _create_ssh_key(os.path.join(ssh_folder, public_key_name)) + + with open(public_key_file) as stream: + public_key = stream.read() + + if project.mirror_hook.public_key != public_key: + _log.info('Updating information in the DB') + project.mirror_hook.public_key = public_key + session.add(project.mirror_hook) + session.commit() + + +@conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True) +@pagure_task +def teardown_mirroring(self, session, username, namespace, name): + ''' Stop the mirroring of the specified project. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + plugin.db_object() + + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=username) + + ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER'] + + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + if os.path.exists(private_key_file): + os.unlink(private_key_file) + + if os.path.exists(public_key_file): + os.unlink(public_key_file) + + project.mirror_hook.public_key = None + session.add(project.mirror_hook) + session.commit() + + +@conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True) +@pagure_task +def mirror_project(self, session, username, namespace, name): + ''' Does the actual mirroring of the specified project. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + plugin.db_object() + + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=username) + + repofolder = pagure_config['GIT_FOLDER'] + repopath = os.path.join(repofolder, project.path) + if not os.path.exists(repopath): + _log.info('Git folder not found at: %s, bailing', repopath) + return + + newpath = tempfile.mkdtemp(prefix='pagure-mirror-') + pygit2.clone_repository(repopath, newpath) + + ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER'] + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + + # Get the list of remotes + remotes = [ + remote.strip() + for remote in project.mirror_hook.target.split('\n') + if project.mirror_hook and remote.strip() + and ssh_urlpattern.match(remote.strip()) + ] + + # Add the remotes + for idx, remote in enumerate(remotes): + remote_name = '%s_%s' % (public_key_name, idx) + _log.info('Adding remote %s as %s', remote, remote_name) + (stdout, stderr) = pagure.lib.git.read_git_lines( + ['remote', 'add', remote_name, remote, '--mirror=push'], + abspath=newpath, error=True) + _log.info( + "Output from git remote add:\n stdout: %s\n stderr: %s", + stdout, stderr) + + # Push + logs = [] + for idx, remote in enumerate(remotes): + remote_name = '%s_%s' % (public_key_name, idx) + _log.info( + 'Pushing to remote %s using key: %s', remote_name, + private_key_file) + (stdout, stderr) = pagure.lib.git.read_git_lines( + ['push', remote_name], + abspath=newpath, error=True, + env={'GIT_SSH_COMMAND': 'ssh -i %s' % private_key_file}) + log = "Output from the push:\n stdout: %s\n stderr: %s" % ( + stdout, stderr) + logs.append(log) + if logs: + project.mirror_hook.last_log = '\n'.join(logs) + session.add(project.mirror_hook) + session.commit() + _log.info('\n'.join(logs)) + + # Remove the clone + shutil.rmtree(newpath) diff --git a/pagure/utils.py b/pagure/utils.py index 1d647d8..101f99a 100644 --- a/pagure/utils.py +++ b/pagure/utils.py @@ -344,6 +344,47 @@ urlregex = re.compile( urlpattern = re.compile(urlregex) +ssh_urlregex = re.compile( + u"^" + # protocol identifier + u"(?:(?:(git\+)?ssh)://)" + # user:pass authentication + u"(?:\S+(?::\S*)?@)?" + u"(?:" + u"(?P" + # IP address exclusion + # private & local networks + u"(?:(?:10|127)" + ip_middle_octet + u"{2}" + ip_last_octet + u")|" + u"(?:(?:169\.254|192\.168)" + ip_middle_octet + ip_last_octet + u")|" + u"(?:172\.(?:1[6-9]|2\d|3[0-1])" + ip_middle_octet + ip_last_octet + u"))" + u"|" + # IP address dotted notation octets + # excludes loopback network 0.0.0.0 + # excludes reserved space >= 224.0.0.0 + # excludes network & broadcast addresses + # (first & last IP address of each class) + u"(?P" + u"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" + u"" + ip_middle_octet + u"{2}" + u"" + ip_last_octet + u")" + u"|" + # host name + u"(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" + # domain name + u"(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*" + # TLD identifier + u"(?:\.(?:[a-z\u00a1-\uffff]{2,}))" + u")" + # port number + u"(?::\d{2,5})?" + # resource path + u"(?:/\S*)?" + u"$", + re.UNICODE | re.IGNORECASE +) +ssh_urlpattern = re.compile(ssh_urlregex) + + def get_repo_path(repo): """ Return the path of the git repository corresponding to the provided Repository object from the DB. diff --git a/tests/__init__.py b/tests/__init__.py index c325303..8c2e5ef 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -362,6 +362,7 @@ class SimplePagureTest(unittest.TestCase): pagure_config.update(reload_config()) imp.reload(pagure.lib.tasks) + imp.reload(pagure.lib.tasks_mirror) imp.reload(pagure.lib.tasks_services) self._app = pagure.flask_app.create_app({'DB_URL': self.dbpath}) diff --git a/tests/test_pagure_flask_ui_plugins.py b/tests/test_pagure_flask_ui_plugins.py index ab5f474..1c68bbf 100644 --- a/tests/test_pagure_flask_ui_plugins.py +++ b/tests/test_pagure_flask_ui_plugins.py @@ -50,6 +50,7 @@ class PagureFlaskPluginstests(tests.SimplePagureTest): 'Fedmsg', 'IRC', 'Mail', + 'Mirroring', 'Pagure', 'Pagure CI', 'Pagure requests', diff --git a/tests/test_pagure_flask_ui_plugins_mirror.py b/tests/test_pagure_flask_ui_plugins_mirror.py new file mode 100644 index 0000000..6ba8ef4 --- /dev/null +++ b/tests/test_pagure_flask_ui_plugins_mirror.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +__requires__ = ['SQLAlchemy >= 0.8'] + +import unittest +import sys +import os + +from mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure.lib +import tests + + +class PagureFlaskPluginMirrortests(tests.Modeltests): + """ Tests for mirror plugin of pagure """ + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureFlaskPluginMirrortests, self).setUp() + + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos')) + + def test_plugin_mirror_no_csrf(self): + """ Test setting up the mirror plugin with no csrf. """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + output = self.app.get('/test/settings/Mirroring') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + data = {} + + output = self.app.post('/test/settings/Mirroring', data=data) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + def test_plugin_mirror_no_data(self): + """ Test the setting up the mirror plugin when there are no data + provided in the request. + """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + data = {'csrf_token': csrf_token} + + # With the git repo + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings - test - Pagure', output_text) + self.assertIn( + ' Hook Mirroring deactivated', + output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + def test_plugin_mirror_invalid_target(self): + """ Test the setting up the mirror plugin when there are the target + provided is invalid. + """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + data = { + 'csrf_token': csrf_token, + 'active': True, + 'target': 'https://host.org/target', + } + + # With the git repo + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + if self.get_wtforms_version() >= (2, 2): + self.assertIn( + '' + '\nInvalid input.', + output_text) + else: + self.assertIn( + '\n' + 'Invalid input.', output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + def test_setting_up_mirror(self): + """ Test the setting up the mirror plugin. + """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + data = { + 'csrf_token': csrf_token, + 'active': True, + 'target': 'ssh://user@host.org/target', + } + + # With the git repo + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings - test - Pagure', + output_text) + self.assertIn( + ' Hook Mirroring activated', + output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertTrue(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + self.assertTrue(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive'))) + + def test_plugin_mirror_deactivate(self): + """ Test the deactivating the mirror plugin. + """ + self.test_setting_up_mirror() + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + # De-Activate hook + data = {'csrf_token': csrf_token} + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings - test - Pagure', + output_text) + self.assertIn( + ' Hook Mirroring deactivated', + output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output.get_data(as_text=True)) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_pagure_flask_util.py b/tests/test_pagure_flask_util.py new file mode 100644 index 0000000..5eec180 --- /dev/null +++ b/tests/test_pagure_flask_util.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +__requires__ = ['SQLAlchemy >= 0.8'] +import pkg_resources + +import unittest +import sys +import os + +from mock import patch, MagicMock + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +from pagure.utils import ssh_urlpattern +import tests + + +class PagureUtilSSHPatterntests(tests.Modeltests): + """ Tests for the ssh_urlpattern in pagure.util """ + + def test_ssh_pattern_valid(self): + """ Test the ssh_urlpattern with valid patterns. """ + patterns = [ + 'ssh://user@host.com/repo.git', + 'git+ssh://user@host.com/repo.git', + 'ssh://host.com/repo.git' + 'git+ssh://host.com/repo.git', + 'ssh://127.0.0.1/repo.git', + 'git+ssh://127.0.0.1/repo.git', + ] + for pattern in patterns: + print(pattern) + self.assertTrue(ssh_urlpattern.match(pattern)) + + + def test_ssh_pattern_invalid(self): + """ Test the ssh_urlpattern with invalid patterns. """ + patterns = [ + 'http://user@host.com/repo.git', + 'git+http://user@host.com/repo.git', + 'https://user@host.com/repo.git', + 'git+https://user@host.com/repo.git', + 'ssh://localhost/repo.git', + 'git+ssh://localhost/repo.git', + 'ssh://0.0.0.0/repo.git', + 'git+ssh://0.0.0.0/repo.git', + ] + for pattern in patterns: + print(pattern) + self.assertFalse(ssh_urlpattern.match(pattern)) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_pagure_lib_task_mirror.py b/tests/test_pagure_lib_task_mirror.py new file mode 100644 index 0000000..f4752df --- /dev/null +++ b/tests/test_pagure_lib_task_mirror.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +import datetime +import os +import shutil +import sys +import tempfile +import time +import unittest + +import pygit2 +import six +from mock import patch, MagicMock, call + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure +import pagure.lib.git +import tests + +import pagure.lib.tasks_mirror + + +class PagureLibTaskMirrortests(tests.Modeltests): + """ Tests for pagure.lib.task_mirror """ + + maxDiff = None + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureLibTaskMirrortests, self).setUp() + + pagure.config.config['REQUESTS_FOLDER'] = None + self.sshkeydir = os.path.join(self.path, 'sshkeys') + pagure.config.config['MIRROR_SSHKEYS_FOLDER'] = self.sshkeydir + + tests.create_projects(self.session) + + def test_create_ssh_key(self): + """ Test the _create_ssh_key method. """ + # before + self.assertFalse(os.path.exists(self.sshkeydir)) + os.mkdir(self.sshkeydir) + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + + keyfile = os.path.join(self.sshkeydir, 'testkey') + pagure.lib.tasks_mirror._create_ssh_key(keyfile) + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'testkey', u'testkey.pub'] + ) + + def test_setup_mirroring(self): + """ Test the setup_mirroring method. """ + + # before + self.assertFalse(os.path.exists(self.sshkeydir)) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + pagure.lib.tasks_mirror.setup_mirroring( + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + def test_setup_mirroring_ssh_folder_exists_wrong_permissions(self): + """ Test the setup_mirroring method. """ + + os.makedirs(self.sshkeydir) + + # before + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + def test_setup_mirroring_ssh_folder_symlink(self): + """ Test the setup_mirroring method. """ + + os.symlink( + self.path, + self.sshkeydir + ) + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'attachments', u'config', u'forks', u'releases', + u'remotes', u'repos', u'sshkeys'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'attachments', u'config', u'forks', u'releases', + u'remotes', u'repos', u'sshkeys'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + @patch('os.getuid', MagicMock(return_value=450)) + def test_setup_mirroring_ssh_folder_owner(self): + """ Test the setup_mirroring method. """ + os.makedirs(self.sshkeydir, mode=0o700) + + # before + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + +class PagureLibTaskMirrorSetuptests(tests.Modeltests): + """ Tests for pagure.lib.task_mirror """ + + maxDiff = None + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureLibTaskMirrorSetuptests, self).setUp() + + pagure.config.config['REQUESTS_FOLDER'] = None + self.sshkeydir = os.path.join(self.path, 'sshkeys') + pagure.config.config['MIRROR_SSHKEYS_FOLDER'] = self.sshkeydir + + tests.create_projects(self.session) + project = pagure.lib.get_authorized_project(self.session, 'test') + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.target = 'ssh://user@localhost.localdomain/foobar.git' + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + pagure.lib.tasks_mirror.setup_mirroring( + username=None, + namespace=None, + name='test') + + def test_setup_mirroring_twice(self): + """ Test the setup_mirroring method. """ + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + before_key = project.mirror_hook.public_key + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertEqual(project.mirror_hook.public_key, before_key) + + def test_teardown_mirroring(self): + """ Test the teardown_mirroring method. """ + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + pagure.lib.tasks_mirror.teardown_mirroring( + username=None, + namespace=None, + name='test') + + # after + self.session = pagure.lib.create_session(self.dbpath) + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + @patch( + 'tempfile.mkdtemp', + MagicMock(return_value='/tmp/pagure-mirror-fdgqcF')) + @patch('pagure.lib.git.read_git_lines') + def test_mirror_project(self,rgl): + """ Test the mirror_project method. """ + rgl.return_value = ('stdout', 'stderr') + tests.create_projects_git( + os.path.join(self.path, 'repos'), bare=True) + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + pagure.lib.tasks_mirror.mirror_project( + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + calls = [ + call( + [ + u'remote', u'add', u'test_0', + u'ssh://user@localhost.localdomain/foobar.git', + u'--mirror=push' + ], + abspath=u'/tmp/pagure-mirror-fdgqcF', + error=True + ), + call( + [u'push', u'test_0'], + abspath=u'/tmp/pagure-mirror-fdgqcF', + env={ + u'GIT_SSH_COMMAND': u'ssh -i %s/sshkeys/test' % self.path + }, + error=True + ) + ] + + self.assertEqual(rgl.call_count, 2) + self.assertEqual( + calls, + rgl.mock_calls + ) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 3403b887543a3fe22d4252d6ebdf3022df14138b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 23 2018 08:02:43 +0000 Subject: [PATCH 5/7] Properly import and call the tasks_services' tasks Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index fbe4fbf..3b90f9c 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -62,6 +62,7 @@ import pagure.utils from pagure.config import config as pagure_config from pagure.lib import model from pagure.lib import tasks +from pagure.lib import tasks_services REDIS = None @@ -1314,7 +1315,7 @@ def add_pull_request_comment(session, request, commit, tree_id, filename, and request.project.ci_hook \ and request.project.ci_hook.active_pr \ and not request.project.private: - pagure.lib.tasks_services.trigger_ci_build.delay( + tasks_services.trigger_ci_build.delay( project_name=request.project_from.fullname, cause=request.id, branch=request.branch_from, @@ -1336,7 +1337,7 @@ def add_pull_request_comment(session, request, commit, tree_id, filename, and pagure_config.get('PAGURE_CI_SERVICES') \ and request.project.ci_hook \ and request.project.ci_hook.active_pr: - pagure.lib.tasks_services.trigger_ci_build.delay( + tasks_services.trigger_ci_build.delay( project_name=request.project_from.fullname, cause=request.id, branch=request.branch_from, @@ -1818,7 +1819,7 @@ def new_pull_request(session, branch_from, and request.project.ci_hook \ and request.project.ci_hook.active_pr \ and not request.project.private: - pagure.lib.tasks_services.trigger_ci_build.delay( + tasks_services.trigger_ci_build.delay( project_name=request.project_from.fullname, cause=request.id, branch=request.branch_from, From dccd888f4f03bc1eefce68c3c4532539c0846d53 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 23 2018 08:02:43 +0000 Subject: [PATCH 6/7] Include a pagure_mirror systemd file Signed-off-by: Pierre-Yves Chibon --- diff --git a/files/pagure_mirror.service b/files/pagure_mirror.service new file mode 100644 index 0000000..5013842 --- /dev/null +++ b/files/pagure_mirror.service @@ -0,0 +1,19 @@ +# This is a systemd's service file for the mirroring service, if you change +# the default value of the CI_CELERY_QUEUE configuration key, do not +# forget to edit it in the ExecStart line below + +[Unit] +Description=Pagure service mirroring projects outside of pagure that asked for it +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/bin/celery worker -A pagure.lib.tasks_mirror --loglevel=info -Q pagure_mirror +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +Type=simple +User=mirror +Group=mirror +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 48f13f346ced5d36d46f37c9b61eeb0dc7e373de Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 23 2018 08:02:43 +0000 Subject: [PATCH 7/7] Add a -mirror subpackage to pagure Signed-off-by: Pierre-Yves Chibon --- diff --git a/files/pagure.spec b/files/pagure.spec index 729716a..4d1549d 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -161,6 +161,16 @@ information provided in the JSON blobs that are stored in the tickets (and in the future pull-requests) git repo. +%package mirror +Summary: The mirroring service for pagure +BuildArch: noarch +Requires: %{name} = %{version}-%{release} +%{?systemd_requires} +%description mirror +pagure-mirror is the service mirroring projects that asked for it outside +of this pagure instance. + + %prep %autosetup -p1 @@ -226,6 +236,10 @@ install -p -m 644 files/pagure_logcom.service \ install -p -m 644 files/pagure_loadjson.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_loadjson.service +# Install the systemd file for the mirror service +install -p -m 644 files/pagure_mirror.service \ + $RPM_BUILD_ROOT/%{_unitdir}/pagure_mirror.service + # Install the systemd file for the script sending reminder about API key # expiration install -p -m 644 files/pagure_api_key_expire_mail.service \ @@ -267,6 +281,8 @@ install -p -m 644 pagure-ev/pagure_ev.service \ %systemd_post pagure_logcom.service %post loadjson %systemd_post pagure_loadjson.service +%post mirror +%systemd_post pagure_mirror.service %preun %systemd_preun pagure_worker.service @@ -284,6 +300,8 @@ install -p -m 644 pagure-ev/pagure_ev.service \ %systemd_preun pagure_logcom.service %preun loadjson %systemd_preun pagure_loadjson.service +%preun mirror +%systemd_preun pagure_mirror.service %postun %systemd_postun_with_restart pagure_worker.service @@ -301,6 +319,8 @@ install -p -m 644 pagure-ev/pagure_ev.service \ %systemd_postun_with_restart pagure_logcom.service %postun loadjson %systemd_postun_with_restart pagure_loadjson.service +%postun mirror +%systemd_postun_with_restart pagure_mirror.service %files @@ -363,6 +383,11 @@ install -p -m 644 pagure-ev/pagure_ev.service \ %{_unitdir}/pagure_loadjson.service +%files mirror +%license LICENSE +%{_unitdir}/pagure_mirror.service + + %changelog * Thu Apr 26 2018 Pierre-Yves Chibon - 4.0.1-1 - Update to 4.0.1