From 18b1db6b51b6edeb1a147880e093b603b2711687 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:06:50 +0000 Subject: [PATCH 1/27] Allow plugins to throw PagureException when they are being installed --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 6447ab6..a3a2736 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -144,6 +144,9 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): except FileNotFoundException as err: _log.exception(err) flask.abort(404, 'No git repo found') + except pagure.exceptions.PagureException as msg: + SESSION.rollback() + flask.flash(msg, 'error') else: try: plugin.remove(repo) From b5c33bba913117772f5d6cb77d8e2cb1e15ef688 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:06:50 +0000 Subject: [PATCH 2/27] Allow plugins to throw PagureException when they are being removed --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index a3a2736..22705ce 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -154,6 +154,9 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): except FileNotFoundException as err: _log.exception(err) flask.abort(404, 'No git repo found') + except pagure.exceptions.PagureException as msg: + SESSION.rollback() + flask.flash(msg, 'error') SESSION.commit() From b5e017d07d3e4e51a79abbbbff0a88991cab57c7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:06:50 +0000 Subject: [PATCH 3/27] Start work on the mirroring hook --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py new file mode 100644 index 0000000..3fbc1da --- /dev/null +++ b/pagure/hooks/mirror_hook.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +import base64 +import os + +import sqlalchemy as sa +import pygit2 +import werkzeug +import wtforms + +from Crypto.PublicKey import RSA +from flask.ext import wtf +from sqlalchemy.orm import relation +from sqlalchemy.orm import backref + +from pagure.exceptions import PagureException +from pagure.hooks import BaseHook, RequiredIf +from pagure.lib.model import BASE, Project +from pagure import APP, get_repo_path + +CONFIG_TPL = '''host %(name)s + HostName %(host)s + User %(user)s + IdentityFile ~/.ssh/%(name)s + +''' + + +def split_target(target): + ''' Check if the given target follows the expected model. ''' + if target.startswith('http'): + raise PagureException( + 'Invalid target %s, we only support mirroring via ssh' % target) + + if target.startswith('ssh://'): + target = target.replace('ssh://', '', 1) + target = target.replace('/', ':', 1) + + if not '@' in target: + raise PagureException( + 'No user specified in %s, we were expecting it before a `@`' + % target) + if not ':' in target: + raise PagureException( + 'No path specified in %s, we were expecting it after a `:`' + % target) + user, host_path = target.split('@', 1) + host, path = host_path.split(':', 1) + return user, host, path + + +def create_ssh_key(keyfile): + ''' Create the public and private ssh keys. + + The specified file name will be the private key and the public one will + be in a similar file name ending with a '.pub'. + + ''' + key = RSA.generate(2048) + with open(keyfile, 'w') as stream: + stream.write(key.exportKey('PEM')) + + with open(keyfile + '.pub', 'w') as stream: + stream.write(key.exportKey('OpenSSH')) + + +def check_or_create_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + user, host, path = split_target(target) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path) + 'name': key_name + } + + update = True + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + update = False + + if update: + with open(ssh_config_file, 'a') as stream: + stream.write(ssh_config) + + +def clean_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + user, host, path = split_target(target) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path) + 'name': key_name + } + + data = None + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + data = data.replace(ssh_config, '', 1) + + if data: + with open(ssh_config_file, 'w') as stream: + stream.write(data) + + +class MirrorTable(BASE): + """ Stores information about the mirroring hook deployed on a project. + + Table -- mirror_pagure + """ + + __tablename__ = 'mirror_pagure' + + id = sa.Column(sa.Integer, primary_key=True) + project_id = sa.Column( + sa.Integer, + sa.ForeignKey( + 'projects.id', onupdate='CASCADE', ondelete='CASCADE'), + nullable=False, + unique=True, + index=True) + + active = sa.Column(sa.Boolean, nullable=False, default=False) + + public_key = sa.Column(sa.Text, nullable=True) + target = sa.Column(sa.Text, nullable=True) + + project = relation( + 'Project', remote_side=[Project.id], + backref=backref( + 'mirror_hook', cascade="delete, delete-orphan", + single_parent=True) + ) + + +class MirrorForm(wtf.Form): + ''' Form to configure the mirror hook. ''' + active = wtforms.BooleanField( + 'Active', + [wtforms.validators.Optional()] + ) + + target = wtforms.TextField( + 'Git repo to mirror to', + [RequiredIf('active')] + ) + + public_key = wtforms.TextField( + 'Public SSH key', + [wtforms.validators.Optional()] + ) + + +DESCRIPTION = ''' +Pagure specific hook to add a comment to issues or pull requests if the pushed +commits fix them +or relate to them. This is determined based on the commit message. + +To reference an issue/PR you need to use one of recognized keywords followed by +a reference to the issue or PR, separated by whitespace and and optional colon. +Such references can be either: + + * The issue/PR number preceded by the `#` symbol + * The full URL of the issue or PR + +If using the full URL, it is possible to reference issues in other projects. + +The recognized keywords are: + + * fix/fixed/fixes + * relate/related/relates + * merge/merges/merged + +Examples: + + * Fixes #21 + * related: https://pagure.io/myproject/issue/32 + * this commit merges #74 + * Merged: https://pagure.io/myproject/pull-request/74 + +Capitalization does not matter; neither does the colon between keyword and +number. + + +''' + + +class MirrorHook(BaseHook): + ''' Mirror hook. ''' + + name = 'Mirroring' + description = DESCRIPTION + form = MirrorForm + db_object = MirrorTable + backref = 'mirror_hook' + form_fields = ['active'] + + @classmethod + def install(cls, project, dbobj): + ''' Method called to install the hook for a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + if not APP.config.get('GITOLITE_HOME'): + raise PagureException( + 'Gitolite wrongly configured, please contact your admin.') + + ssh_folder = os.path.join(APP.config.get('GITOLITE_HOME'), '.ssh') + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_name = werkzeug.secure_filename(project.fullname) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + if not os.path.exist(public_key_file): + create_ssh_key(os.path.join(ssh_folder, public_key_name)) + + with open(public_key_file) as stream: + public_key = stream.read() + + check_or_create_ssh_config( + ssh_folder, public_key_name, dbobj.target) + + if dbobj.public_key != public_key: + dbobj.public_key = public_key + APP.SESSION.add(dbobj) + APP.SESSION.commit() + + repopaths = [get_repo_path(project)] + cls.base_install(repopaths, dbobj, 'mirror', 'mirror_hook.py') + + @classmethod + def remove(cls, project): + ''' Method called to remove the hook of a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + if not APP.config.get('GITOLITE_HOME'): + raise PagureException( + 'Gitolite wrongly configured, please contact your admin.') + + ssh_folder = os.path.join(APP.config.get('GITOLITE_HOME'), '.ssh') + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_name = werkzeug.secure_filename(project.fullname) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + if os.path.exist(public_key_file): + os.unlink(public_key_file) + + clean_ssh_config(ssh_folder, key_name, target) + + repopaths = [get_repo_path(project)] + + cls.base_remove(repopaths, 'mirror') From a654a98ee1a5a522541c7bc86f7796e55647a9b5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:16 +0000 Subject: [PATCH 4/27] Fix flashing the PagureException errors --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 22705ce..2a4bd3c 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -144,9 +144,9 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): except FileNotFoundException as err: _log.exception(err) flask.abort(404, 'No git repo found') - except pagure.exceptions.PagureException as msg: + except pagure.exceptions.PagureException as err: SESSION.rollback() - flask.flash(msg, 'error') + flask.flash(str(err), 'error') else: try: plugin.remove(repo) @@ -154,9 +154,9 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): except FileNotFoundException as err: _log.exception(err) flask.abort(404, 'No git repo found') - except pagure.exceptions.PagureException as msg: + except pagure.exceptions.PagureException as err: SESSION.rollback() - flask.flash(msg, 'error') + flask.flash(str(err), 'error') SESSION.commit() From 73da342a140e849b3c98612bfdfa35f753bd746c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:16 +0000 Subject: [PATCH 5/27] Support mirroring to multiple targets with the same key --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 3fbc1da..39008a0 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -24,12 +24,13 @@ from sqlalchemy.orm import backref from pagure.exceptions import PagureException from pagure.hooks import BaseHook, RequiredIf from pagure.lib.model import BASE, Project -from pagure import APP, get_repo_path +from pagure import APP, SESSION, get_repo_path + CONFIG_TPL = '''host %(name)s HostName %(host)s User %(user)s - IdentityFile ~/.ssh/%(name)s + IdentityFile ~/.ssh/%(keyname)s ''' @@ -75,47 +76,61 @@ def create_ssh_key(keyfile): def check_or_create_ssh_config(ssh_folder, key_name, target): ''' Check or adjust the ~/.ssh/config file ''' ssh_config_file = os.path.join(ssh_folder, 'config') - user, host, path = split_target(target) - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path) - 'name': key_name - } + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } - update = True - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - update = False + update = True + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + update = False - if update: - with open(ssh_config_file, 'a') as stream: - stream.write(ssh_config) + if update: + with open(ssh_config_file, 'a') as stream: + stream.write(ssh_config) def clean_ssh_config(ssh_folder, key_name, target): ''' Check or adjust the ~/.ssh/config file ''' ssh_config_file = os.path.join(ssh_folder, 'config') - user, host, path = split_target(target) - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path) - 'name': key_name - } + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue - data = None - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - data = data.replace(ssh_config, '', 1) + user, host, path = split_target(remote) - if data: - with open(ssh_config_file, 'w') as stream: - stream.write(data) + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + data = None + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + data = data.replace(ssh_config, '', 1) + + if data: + with open(ssh_config_file, 'w') as stream: + stream.write(data) class MirrorTable(BASE): @@ -124,7 +139,7 @@ class MirrorTable(BASE): Table -- mirror_pagure """ - __tablename__ = 'mirror_pagure' + __tablename__ = 'hook_mirror' id = sa.Column(sa.Integer, primary_key=True) project_id = sa.Column( @@ -160,42 +175,14 @@ class MirrorForm(wtf.Form): [RequiredIf('active')] ) - public_key = wtforms.TextField( + public_key = wtforms.TextAreaField( 'Public SSH key', [wtforms.validators.Optional()] ) DESCRIPTION = ''' -Pagure specific hook to add a comment to issues or pull requests if the pushed -commits fix them -or relate to them. This is determined based on the commit message. - -To reference an issue/PR you need to use one of recognized keywords followed by -a reference to the issue or PR, separated by whitespace and and optional colon. -Such references can be either: - - * The issue/PR number preceded by the `#` symbol - * The full URL of the issue or PR - -If using the full URL, it is possible to reference issues in other projects. - -The recognized keywords are: - - * fix/fixed/fixes - * relate/related/relates - * merge/merges/merged - -Examples: - - * Fixes #21 - * related: https://pagure.io/myproject/issue/32 - * this commit merges #74 - * Merged: https://pagure.io/myproject/pull-request/74 - -Capitalization does not matter; neither does the colon between keyword and -number. - +Pagure specific hook to mirror a repo hosted on pagure to another location. ''' @@ -208,7 +195,7 @@ class MirrorHook(BaseHook): form = MirrorForm db_object = MirrorTable backref = 'mirror_hook' - form_fields = ['active'] + form_fields = ['active', 'target', 'public_key'] @classmethod def install(cls, project, dbobj): @@ -227,24 +214,26 @@ class MirrorHook(BaseHook): os.makedirs(ssh_folder) public_key_name = werkzeug.secure_filename(project.fullname) + public_key_file = os.path.join( ssh_folder, '%s.pub' % public_key_name) - if not os.path.exist(public_key_file): - create_ssh_key(os.path.join(ssh_folder, public_key_name)) - with open(public_key_file) as stream: - public_key = stream.read() + if not os.path.exists(public_key_file): + create_ssh_key(os.path.join(ssh_folder, public_key_name)) check_or_create_ssh_config( ssh_folder, public_key_name, dbobj.target) + with open(public_key_file) as stream: + public_key = stream.read() + if dbobj.public_key != public_key: dbobj.public_key = public_key - APP.SESSION.add(dbobj) - APP.SESSION.commit() + SESSION.add(dbobj) + SESSION.commit() repopaths = [get_repo_path(project)] - cls.base_install(repopaths, dbobj, 'mirror', 'mirror_hook.py') + cls.base_install(repopaths, dbobj, 'mirror', 'mirror.py') @classmethod def remove(cls, project): @@ -266,7 +255,7 @@ class MirrorHook(BaseHook): public_key_file = os.path.join( ssh_folder, '%s.pub' % public_key_name) - if os.path.exist(public_key_file): + if os.path.exists(public_key_file): os.unlink(public_key_file) clean_ssh_config(ssh_folder, key_name, target) From 89546419344548e2b687a572cd2dba60a49ef53f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:16 +0000 Subject: [PATCH 6/27] Add the actual mirror hook --- diff --git a/pagure/hooks/files/mirror.py b/pagure/hooks/files/mirror.py new file mode 100755 index 0000000..aae03e1 --- /dev/null +++ b/pagure/hooks/files/mirror.py @@ -0,0 +1,73 @@ +#! /usr/bin/env python2 + + +"""Pagure specific hook to mirror a repo to another location. +""" + +import os +import sys + +import werkzeug + +from sqlalchemy.exc import SQLAlchemyError + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.exceptions +import pagure.lib +import pagure.ui.plugins + + +abspath = os.path.abspath(os.environ['GIT_DIR']) + + +def mirror_repo(): + + reponame = pagure.lib.git.get_repo_name(abspath) + username = pagure.lib.git.get_username(abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print 'repo:', reponame, username + + repo = pagure.lib.get_project(pagure.SESSION, reponame, user=username) + if not repo: + print 'Unknown repo %s of username: %s' % (reponame, username) + sys.exit(1) + + plugin = pagure.ui.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + + # Get the list of remotes + remotes = [ + remote.strip() + for remote in repo.mirror_hook[0].target.split('\n') + if repo.mirror_hook and remote.strip() + ] + + public_key_name = werkzeug.secure_filename(repo.fullname) + + # Add the remotes + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, + '--mirror=push'], abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + # Push + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['push', '%s_%s' % (public_key_name, idx)], abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + +def main(args): + mirror_repo() + + +if __name__ == '__main__': + main(sys.argv[1:]) From 2e6f478a665f021602cc7622d2443b2a3bebf087 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:16 +0000 Subject: [PATCH 7/27] Move from crypto to cryptography since we're using it elsewhere --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 39008a0..bcb531f 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -16,7 +16,10 @@ import pygit2 import werkzeug import wtforms -from Crypto.PublicKey import RSA +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + from flask.ext import wtf from sqlalchemy.orm import relation from sqlalchemy.orm import backref @@ -65,10 +68,25 @@ def create_ssh_key(keyfile): be in a similar file name ending with a '.pub'. ''' - key = RSA.generate(2048) + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + + pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) with open(keyfile, 'w') as stream: - stream.write(key.exportKey('PEM')) + stream.write(pem) + public_key = private_key.public_key() + pem = public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) with open(keyfile + '.pub', 'w') as stream: stream.write(key.exportKey('OpenSSH')) From 07506c07eafc697c7c7fbdc4f18144c7c078af73 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 8/27] Make of cryptography a full requirement --- diff --git a/requirements.txt b/requirements.txt index 14d115e..4257e36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ binaryornot < 0.4.3 bleach < 2.0 blinker chardet < 3.0.0 +cryptography docutils enum34 flask From ca6257cf8a82b6bb43f3b1c16373eee87e0f6fd1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 9/27] Rename the variable to avoid any error and fix writing the public key --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index bcb531f..c4780cf 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -74,21 +74,21 @@ def create_ssh_key(keyfile): backend=default_backend() ) - pem = private_key.private_bytes( + private_pem = private_key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm=serialization.NoEncryption() ) with open(keyfile, 'w') as stream: - stream.write(pem) + stream.write(private_pem) public_key = private_key.public_key() - pem = public_key.public_bytes( + public_pem = public_key.public_bytes( encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo ) with open(keyfile + '.pub', 'w') as stream: - stream.write(key.exportKey('OpenSSH')) + stream.write(public_pem) def check_or_create_ssh_config(ssh_folder, key_name, target): From 8a72ff49780639415d89d105468381d5ad25a948 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 10/27] Fix unit-tests for the new plugin added --- diff --git a/tests/test_pagure_flask_ui_plugins.py b/tests/test_pagure_flask_ui_plugins.py index d572f1c..f8e9050 100644 --- a/tests/test_pagure_flask_ui_plugins.py +++ b/tests/test_pagure_flask_ui_plugins.py @@ -66,12 +66,12 @@ class PagureFlaskPluginstests(tests.Modeltests): """ Test the get_plugin_names function. """ names = pagure.lib.plugins.get_plugin_names() self.assertEqual( - sorted(names), [ 'Block Un-Signed commits', 'Block non fast-forward pushes', - 'Fedmsg', 'IRC', 'Mail', 'Pagure', 'Pagure CI', + 'Fedmsg', 'IRC', 'Mail', 'Mirroring', 'Pagure', 'Pagure CI', 'Pagure requests', 'Pagure tickets', 'Read the Doc', - ] + ], + sorted(names) ) def test_get_plugin(self): From e0df2a8b597549b606deed0702fe6b86f4eacdb7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 11/27] Backport from a newercrytography the method to serialise public ssh key This allows converting a public key in a format that makes it available to use by OpenSSH --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index c4780cf..a7abcd9 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -10,12 +10,15 @@ import base64 import os +import struct import sqlalchemy as sa +import six import pygit2 import werkzeug import wtforms +from cryptography import utils from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization @@ -38,6 +41,60 @@ CONFIG_TPL = '''host %(name)s ''' +# Code from: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 +def _ssh_write_string(data): + return struct.pack(">I", len(data)) + data + + +def _ssh_write_mpint(value): + data = utils.int_to_bytes(value) + if six.indexbytes(data, 0) & 0x80: + data = b"\x00" + data + return _ssh_write_string(data) + + +# Code from: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 +def serialize_public_ssh_key( key): + if isinstance(key, rsa.RSAPublicKey): + public_numbers = key.public_numbers() + return b"ssh-rsa " + base64.b64encode( + _ssh_write_string(b"ssh-rsa") + + _ssh_write_mpint(public_numbers.e) + + _ssh_write_mpint(public_numbers.n) + ) + elif isinstance(key, dsa.DSAPublicKey): + public_numbers = key.public_numbers() + parameter_numbers = public_numbers.parameter_numbers + return b"ssh-dss " + base64.b64encode( + _ssh_write_string(b"ssh-dss") + + _ssh_write_mpint(parameter_numbers.p) + + _ssh_write_mpint(parameter_numbers.q) + + _ssh_write_mpint(parameter_numbers.g) + + _ssh_write_mpint(public_numbers.y) + ) + else: + assert isinstance(key, ec.EllipticCurvePublicKey) + public_numbers = key.public_numbers() + try: + curve_name = { + ec.SECP256R1: b"nistp256", + ec.SECP384R1: b"nistp384", + ec.SECP521R1: b"nistp521", + }[type(public_numbers.curve)] + except KeyError: + raise ValueError( + "Only SECP256R1, SECP384R1, and SECP521R1 curves are " + "supported by the SSH public key format" + ) + return b"ecdsa-sha2-" + curve_name + b" " + base64.b64encode( + _ssh_write_string(b"ecdsa-sha2-" + curve_name) + + _ssh_write_string(curve_name) + + _ssh_write_string(public_numbers.encode_point()) + ) + + def split_target(target): ''' Check if the given target follows the expected model. ''' if target.startswith('http'): @@ -83,10 +140,7 @@ def create_ssh_key(keyfile): stream.write(private_pem) public_key = private_key.public_key() - public_pem = public_key.public_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PublicFormat.SubjectPublicKeyInfo - ) + public_pem = serialize_public_ssh_key(public_key) with open(keyfile + '.pub', 'w') as stream: stream.write(public_pem) From 8e55810a0128080d91b77be19e6a8d244b9f8bef Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 12/27] Make the backref from project to mirror_hook to point to only 1 project --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index a7abcd9..6c4b795 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -231,7 +231,7 @@ class MirrorTable(BASE): 'Project', remote_side=[Project.id], backref=backref( 'mirror_hook', cascade="delete, delete-orphan", - single_parent=True) + single_parent=True, uselist=False) ) From cbafc9f583819aef8cd99427ed23b665e4d05906 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 13/27] Fix uninstalling the mirroring hook Remove the private key when removing the mirror hook Clean the ~/.ssh/config file for this project --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 6c4b795..6cccf53 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -324,13 +324,17 @@ class MirrorHook(BaseHook): os.makedirs(ssh_folder) public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) public_key_file = os.path.join( ssh_folder, '%s.pub' % public_key_name) + if os.path.exists(private_key_file): + os.unlink(private_key_file) + if os.path.exists(public_key_file): os.unlink(public_key_file) - clean_ssh_config(ssh_folder, key_name, target) + clean_ssh_config(ssh_folder, public_key_name, project.mirror_hook.target) repopaths = [get_repo_path(project)] From 822d2b4baeb047e801caa518c1fc6af91cdfb2b5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 14/27] Drop the other OpenSSL keys from the serialization since we only use RSA --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 6cccf53..bb78a7c 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -54,7 +54,7 @@ def _ssh_write_mpint(value): return _ssh_write_string(data) -# Code from: +# Code from _openssh_public_key_bytes at: # https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 def serialize_public_ssh_key( key): if isinstance(key, rsa.RSAPublicKey): @@ -64,35 +64,9 @@ def serialize_public_ssh_key( key): _ssh_write_mpint(public_numbers.e) + _ssh_write_mpint(public_numbers.n) ) - elif isinstance(key, dsa.DSAPublicKey): - public_numbers = key.public_numbers() - parameter_numbers = public_numbers.parameter_numbers - return b"ssh-dss " + base64.b64encode( - _ssh_write_string(b"ssh-dss") + - _ssh_write_mpint(parameter_numbers.p) + - _ssh_write_mpint(parameter_numbers.q) + - _ssh_write_mpint(parameter_numbers.g) + - _ssh_write_mpint(public_numbers.y) - ) else: - assert isinstance(key, ec.EllipticCurvePublicKey) - public_numbers = key.public_numbers() - try: - curve_name = { - ec.SECP256R1: b"nistp256", - ec.SECP384R1: b"nistp384", - ec.SECP521R1: b"nistp521", - }[type(public_numbers.curve)] - except KeyError: - raise ValueError( - "Only SECP256R1, SECP384R1, and SECP521R1 curves are " - "supported by the SSH public key format" - ) - return b"ecdsa-sha2-" + curve_name + b" " + base64.b64encode( - _ssh_write_string(b"ecdsa-sha2-" + curve_name) + - _ssh_write_string(curve_name) + - _ssh_write_string(public_numbers.encode_point()) - ) + # Since we only write RSA keys, drop the other serializations + return def split_target(target): @@ -141,8 +115,9 @@ def create_ssh_key(keyfile): public_key = private_key.public_key() public_pem = serialize_public_ssh_key(public_key) - with open(keyfile + '.pub', 'w') as stream: - stream.write(public_pem) + if public_pem: + with open(keyfile + '.pub', 'w') as stream: + stream.write(public_pem) def check_or_create_ssh_config(ssh_folder, key_name, target): From 3058c1bc970ce039a1b2822b02c070e24b35e170 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 15/27] Always write the ~/.ssh/config file Otherwise we cannot clean the last entry in the file. --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index bb78a7c..9dd4e8e 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -175,9 +175,9 @@ def clean_ssh_config(ssh_folder, key_name, target): if ssh_config in data: data = data.replace(ssh_config, '', 1) - if data: - with open(ssh_config_file, 'w') as stream: - stream.write(data) + + with open(ssh_config_file, 'w') as stream: + stream.write(data) class MirrorTable(BASE): From 98b5aa3c2cc6563e8afe8e34f11a82dbf1bee00d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 16/27] Clean the public_key in the DB when removing the mirroring hook This is needed since we also remove the files. --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 9dd4e8e..5685d6a 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -314,3 +314,7 @@ class MirrorHook(BaseHook): repopaths = [get_repo_path(project)] cls.base_remove(repopaths, 'mirror') + + project.mirror_hook.public_key = None + SESSION.add(project) + SESSION.commit() From 6a93ff98395e19e90e3e0463dea0cea64c3773fd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 17/27] Add a new service: pagure-mirror This service will allow us to decouple mirroring git repos from the main user running the pagure app. For security reasons, this is good! --- diff --git a/pagure-mirror/pagure_ci_server.py b/pagure-mirror/pagure_ci_server.py new file mode 100644 index 0000000..47a09ed --- /dev/null +++ b/pagure-mirror/pagure_ci_server.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and set-up/remove mirroring +for the corresponding project. + +""" + +import base64 +import json +import logging +import os +import struct + +import requests +import six +import trollius +import trollius_redis +import werkzeug + +from cryptography import utils +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + + + +logging.basicConfig(level=logging.DEBUG) +LOG = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +CONFIG_TPL = '''host %(name)s + HostName %(host)s + User %(user)s + IdentityFile ~/.ssh/%(keyname)s + +''' + + +# +# Utility methods used to setup/teardown the mirroring +# + + +# Code from: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 +def _ssh_write_string(data): + return struct.pack(">I", len(data)) + data + + +def _ssh_write_mpint(value): + data = utils.int_to_bytes(value) + if six.indexbytes(data, 0) & 0x80: + data = b"\x00" + data + return _ssh_write_string(data) + + +# Code from _openssh_public_key_bytes at: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 +@trollius.coroutine +def serialize_public_ssh_key( key): + if isinstance(key, rsa.RSAPublicKey): + public_numbers = key.public_numbers() + return b"ssh-rsa " + base64.b64encode( + _ssh_write_string(b"ssh-rsa") + + _ssh_write_mpint(public_numbers.e) + + _ssh_write_mpint(public_numbers.n) + ) + else: + # Since we only write RSA keys, drop the other serializations + return + + +@trollius.coroutine +def create_ssh_key(keyfile): + ''' Create the public and private ssh keys. + + The specified file name will be the private key and the public one will + be in a similar file name ending with a '.pub'. + + ''' + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + with open(keyfile, 'w') as stream: + stream.write(private_pem) + + public_key = private_key.public_key() + public_pem = yield trollius.From(serialize_public_ssh_key(public_key)) + if public_pem: + with open(keyfile + '.pub', 'w') as stream: + stream.write(public_pem) + + +def split_target(target): + ''' Check if the given target follows the expected model. ''' + LOG.info('Checking target: %s', target) + if target.startswith('http'): + raise PagureException( + 'Invalid target %s, we only support mirroring via ssh' % target) + + if target.startswith('ssh://'): + target = target.replace('ssh://', '', 1) + target = target.replace('/', ':', 1) + + if not '@' in target: + raise PagureException( + 'No user specified in %s, we were expecting it before a `@`' + % target) + if not ':' in target: + raise PagureException( + 'No path specified in %s, we were expecting it after a `:`' + % target) + user, host_path = target.split('@', 1) + host, path = host_path.split(':', 1) + return user, host, path + + +@trollius.coroutine +def check_or_create_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + + + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + update = True + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + update = False + + if update: + with open(ssh_config_file, 'a') as stream: + stream.write(ssh_config) + + +@trollius.coroutine +def clean_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + data = None + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + data = data.replace(ssh_config, '', 1) + + with open(ssh_config_file, 'w') as stream: + stream.write(data) + + +# +# Actual logic of the service +# + +@trollius.coroutine +def setup_mirroring(project, session, dbobj): + ''' Setup the specified repo for mirroring. + ''' + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + LOG.info('Public key of interest: %s', public_key_file) + + if not os.path.exists(public_key_file): + LOG.info('Creating public key') + yield trollius.From( + create_ssh_key(os.path.join(ssh_folder, public_key_name)) + ) + + LOG.info('Updating ssh configuration') + yield trollius.From( + check_or_create_ssh_config( + ssh_folder, public_key_name, dbobj.target) + ) + + with open(public_key_file) as stream: + public_key = stream.read() + + if dbobj.public_key != public_key: + LOG.info('Updating information in the DB') + dbobj.public_key = public_key + session.add(dbobj) + session.commit() + + +@trollius.coroutine +def mirror_project(repo, dbobj): + ''' Does the actual mirroring of the specified project/repo. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + + # Get the list of remotes + remotes = [ + remote.strip() + for remote in repo.mirror_hook[0].target.split('\n') + if repo.mirror_hook and remote.strip() + ] + + public_key_name = werkzeug.secure_filename(repo.fullname) + + # Add the remotes + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, + '--mirror=push'], abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + # Push + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['push', '%s_%s' % (public_key_name, idx)], abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + +@trollius.coroutine +def teardown_mirroring(project, session, dbobj): + ''' Stop the mirroring of the specified repo. + ''' + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + if os.path.exists(private_key_file): + os.unlink(private_key_file) + + if os.path.exists(public_key_file): + os.unlink(public_key_file) + + yield trollius.From( + clean_ssh_config( + ssh_folder, public_key_name, dbobj.target) + ) + + project.mirror_hook.public_key = None + session.add(project) + session.commit() + + +@trollius.coroutine +def handle_messages(): + ''' Handles connecting to redis and acting upon messages received. + In this case, it means triggering a build on jenkins based on the + information provided. + ''' + + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.mirror'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + LOG.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + reponame = data['name'] + username = data['user']['name'] if data['parent'] else None + namespace = data['namespace'] + LOG.info( + 'Looking for project: %s/%s/%s', namespace, username, reponame) + + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + repo = pagure.lib.get_project( + session, reponame, + user=username, + namespace=namespace) + if not repo: + print 'Unknown repo %s of username: %s in ns: %s' % ( + reponame, username, namespace) + session.close() + sys.exit(1) + + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj = getattr(repo, plugin.backref) + + topic = data.get('topic') + if topic == 'pagure.mirror.postcommit': + yield trollius.From(mirror_project(repo, dbobj)) + elif topic == 'pagure.mirror.setup': + yield trollius.From(setup_mirroring(repo, session, dbobj)) + elif topic == 'pagure.mirror.teardown': + yield trollius.From(teardown_mirroring(repo, session, dbobj)) + else: + LOG.error('Unknown topic found: %s', topic) + + session.close() + LOG.info('Ready for another') + + +def main(): + ''' Start the main async loop. ''' + + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + LOG.info("End Connection") + loop.close() + LOG.info("End") + + +if __name__ == '__main__': + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + LOG.setLevel(logging.DEBUG) + shellhandler = logging.StreamHandler() + shellhandler.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + aslog = logging.getLogger("trollius") + aslog.setLevel(logging.DEBUG) + + shellhandler.setFormatter(formatter) + LOG.addHandler(shellhandler) + main() From 76b6f121a132a8b391c7e31939b1203471e6a0b0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 18/27] Adjust the mirror_hook plugins to work together with pagure-mirror This way the logic is in the service receiving the redis notifications and not in the main pagure application. Allowing these two to be run by two different users. --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 5685d6a..144043d 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -8,9 +8,8 @@ """ -import base64 +import json import os -import struct import sqlalchemy as sa import six @@ -18,168 +17,17 @@ import pygit2 import werkzeug import wtforms -from cryptography import utils -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives import serialization - from flask.ext import wtf from sqlalchemy.orm import relation from sqlalchemy.orm import backref from pagure.exceptions import PagureException from pagure.hooks import BaseHook, RequiredIf +from pagure.lib import REDIS from pagure.lib.model import BASE, Project from pagure import APP, SESSION, get_repo_path -CONFIG_TPL = '''host %(name)s - HostName %(host)s - User %(user)s - IdentityFile ~/.ssh/%(keyname)s - -''' - - -# Code from: -# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 -def _ssh_write_string(data): - return struct.pack(">I", len(data)) + data - - -def _ssh_write_mpint(value): - data = utils.int_to_bytes(value) - if six.indexbytes(data, 0) & 0x80: - data = b"\x00" + data - return _ssh_write_string(data) - - -# Code from _openssh_public_key_bytes at: -# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 -def serialize_public_ssh_key( key): - if isinstance(key, rsa.RSAPublicKey): - public_numbers = key.public_numbers() - return b"ssh-rsa " + base64.b64encode( - _ssh_write_string(b"ssh-rsa") + - _ssh_write_mpint(public_numbers.e) + - _ssh_write_mpint(public_numbers.n) - ) - else: - # Since we only write RSA keys, drop the other serializations - return - - -def split_target(target): - ''' Check if the given target follows the expected model. ''' - if target.startswith('http'): - raise PagureException( - 'Invalid target %s, we only support mirroring via ssh' % target) - - if target.startswith('ssh://'): - target = target.replace('ssh://', '', 1) - target = target.replace('/', ':', 1) - - if not '@' in target: - raise PagureException( - 'No user specified in %s, we were expecting it before a `@`' - % target) - if not ':' in target: - raise PagureException( - 'No path specified in %s, we were expecting it after a `:`' - % target) - user, host_path = target.split('@', 1) - host, path = host_path.split(':', 1) - return user, host, path - - -def create_ssh_key(keyfile): - ''' Create the public and private ssh keys. - - The specified file name will be the private key and the public one will - be in a similar file name ending with a '.pub'. - - ''' - private_key = rsa.generate_private_key( - public_exponent=65537, - key_size=4096, - backend=default_backend() - ) - - private_pem = private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption() - ) - with open(keyfile, 'w') as stream: - stream.write(private_pem) - - public_key = private_key.public_key() - public_pem = serialize_public_ssh_key(public_key) - if public_pem: - with open(keyfile + '.pub', 'w') as stream: - stream.write(public_pem) - - -def check_or_create_ssh_config(ssh_folder, key_name, target): - ''' Check or adjust the ~/.ssh/config file ''' - ssh_config_file = os.path.join(ssh_folder, 'config') - - for idx, remote in enumerate(target.split('\n')): - remote = remote.strip() - if not remote: - continue - - user, host, path = split_target(remote) - - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path), - 'name': '%s_%s' % (key_name, idx), - 'keyname': key_name, - } - - update = True - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - update = False - - if update: - with open(ssh_config_file, 'a') as stream: - stream.write(ssh_config) - - -def clean_ssh_config(ssh_folder, key_name, target): - ''' Check or adjust the ~/.ssh/config file ''' - ssh_config_file = os.path.join(ssh_folder, 'config') - - for idx, remote in enumerate(target.split('\n')): - remote = remote.strip() - if not remote: - continue - - user, host, path = split_target(remote) - - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path), - 'name': '%s_%s' % (key_name, idx), - 'keyname': key_name, - } - - data = None - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - data = data.replace(ssh_config, '', 1) - - - with open(ssh_config_file, 'w') as stream: - stream.write(data) - - class MirrorTable(BASE): """ Stores information about the mirroring hook deployed on a project. @@ -252,32 +100,9 @@ class MirrorHook(BaseHook): should be installed ''' - if not APP.config.get('GITOLITE_HOME'): - raise PagureException( - 'Gitolite wrongly configured, please contact your admin.') - - ssh_folder = os.path.join(APP.config.get('GITOLITE_HOME'), '.ssh') - if not os.path.exists(ssh_folder): - os.makedirs(ssh_folder) - - public_key_name = werkzeug.secure_filename(project.fullname) - - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - - if not os.path.exists(public_key_file): - create_ssh_key(os.path.join(ssh_folder, public_key_name)) - - check_or_create_ssh_config( - ssh_folder, public_key_name, dbobj.target) - - with open(public_key_file) as stream: - public_key = stream.read() - - if dbobj.public_key != public_key: - dbobj.public_key = public_key - SESSION.add(dbobj) - SESSION.commit() + data = project.to_json(public=True) + data['topic'] = 'pagure.mirror.setup' + REDIS.publish('pagure.mirror', json.dumps(data)) repopaths = [get_repo_path(project)] cls.base_install(repopaths, dbobj, 'mirror', 'mirror.py') @@ -290,31 +115,9 @@ class MirrorHook(BaseHook): should be installed ''' - if not APP.config.get('GITOLITE_HOME'): - raise PagureException( - 'Gitolite wrongly configured, please contact your admin.') - - ssh_folder = os.path.join(APP.config.get('GITOLITE_HOME'), '.ssh') - if not os.path.exists(ssh_folder): - os.makedirs(ssh_folder) - - public_key_name = werkzeug.secure_filename(project.fullname) - private_key_file = os.path.join(ssh_folder, public_key_name) - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - - if os.path.exists(private_key_file): - os.unlink(private_key_file) - - if os.path.exists(public_key_file): - os.unlink(public_key_file) - - clean_ssh_config(ssh_folder, public_key_name, project.mirror_hook.target) + data = project.to_json(public=True) + data['topic'] = 'pagure.mirror.teardown' + REDIS.publish('pagure.mirror', json.dumps(data)) repopaths = [get_repo_path(project)] - cls.base_remove(repopaths, 'mirror') - - project.mirror_hook.public_key = None - SESSION.add(project) - SESSION.commit() From 4a13ebcdcfe63eb8725004c4ac1b451906c57455 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 19/27] Adjust the pagure_mirror hook to include a last_log field In this field we will save the output of the last git push to help our users diagnose potential issues in the syncing process. --- diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py index 144043d..7261fdd 100644 --- a/pagure/hooks/mirror_hook.py +++ b/pagure/hooks/mirror_hook.py @@ -49,6 +49,7 @@ class MirrorTable(BASE): public_key = sa.Column(sa.Text, nullable=True) target = sa.Column(sa.Text, nullable=True) + last_log = sa.Column(sa.Text, nullable=True) project = relation( 'Project', remote_side=[Project.id], @@ -74,6 +75,10 @@ class MirrorForm(wtf.Form): 'Public SSH key', [wtforms.validators.Optional()] ) + last_log = wtforms.TextAreaField( + 'Log of the last sync:', + [wtforms.validators.Optional()] + ) DESCRIPTION = ''' @@ -90,7 +95,7 @@ class MirrorHook(BaseHook): form = MirrorForm db_object = MirrorTable backref = 'mirror_hook' - form_fields = ['active', 'target', 'public_key'] + form_fields = ['active', 'target', 'public_key', 'last_log'] @classmethod def install(cls, project, dbobj): From 3b7eb5e7c7ac866f92d4a3a405227fa87616185f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 20/27] Adjust the pagure-ci service to save the push log in the DB --- diff --git a/pagure-mirror/pagure_ci_server.py b/pagure-mirror/pagure_ci_server.py index 47a09ed..6c6a73c 100644 --- a/pagure-mirror/pagure_ci_server.py +++ b/pagure-mirror/pagure_ci_server.py @@ -244,7 +244,7 @@ def setup_mirroring(project, session, dbobj): @trollius.coroutine -def mirror_project(repo, dbobj): +def mirror_project(repo, session, dbobj): ''' Does the actual mirroring of the specified project/repo. ''' plugin = pagure.lib.plugins.get_plugin('Mirroring') @@ -271,6 +271,9 @@ def mirror_project(repo, dbobj): for idx, remote in enumerate(remotes): lines = pagure.lib.git.read_git_lines( ['push', '%s_%s' % (public_key_name, idx)], abspath) + dbobj.last_log = '\n'.join(lines) + session.add(dbobj) + session.commit() if pagure.APP.config.get('HOOK_DEBUG', False): print '\n'.join(lines) @@ -359,7 +362,7 @@ def handle_messages(): topic = data.get('topic') if topic == 'pagure.mirror.postcommit': - yield trollius.From(mirror_project(repo, dbobj)) + yield trollius.From(mirror_project(repo, session, dbobj)) elif topic == 'pagure.mirror.setup': yield trollius.From(setup_mirroring(repo, session, dbobj)) elif topic == 'pagure.mirror.teardown': From 4855eee4802b38e02f62c15fb3a6f79704397c3a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 21/27] Retrieve the absolute path to the repo sent by the server --- diff --git a/pagure-mirror/pagure_ci_server.py b/pagure-mirror/pagure_ci_server.py index 6c6a73c..215112e 100644 --- a/pagure-mirror/pagure_ci_server.py +++ b/pagure-mirror/pagure_ci_server.py @@ -244,7 +244,7 @@ def setup_mirroring(project, session, dbobj): @trollius.coroutine -def mirror_project(repo, session, dbobj): +def mirror_project(repo, session, dbobj, abspath): ''' Does the actual mirroring of the specified project/repo. ''' plugin = pagure.lib.plugins.get_plugin('Mirroring') @@ -253,7 +253,7 @@ def mirror_project(repo, session, dbobj): # Get the list of remotes remotes = [ remote.strip() - for remote in repo.mirror_hook[0].target.split('\n') + for remote in repo.mirror_hook.target.split('\n') if repo.mirror_hook and remote.strip() ] @@ -342,6 +342,7 @@ def handle_messages(): reponame = data['name'] username = data['user']['name'] if data['parent'] else None namespace = data['namespace'] + abspath = data['abspath'] LOG.info( 'Looking for project: %s/%s/%s', namespace, username, reponame) @@ -362,7 +363,7 @@ def handle_messages(): topic = data.get('topic') if topic == 'pagure.mirror.postcommit': - yield trollius.From(mirror_project(repo, session, dbobj)) + yield trollius.From(mirror_project(repo, session, dbobj, abspath)) elif topic == 'pagure.mirror.setup': yield trollius.From(setup_mirroring(repo, session, dbobj)) elif topic == 'pagure.mirror.teardown': From 9c66762227341710a7b2b6dd280be52a9417c03c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 22/27] Improve the mirror hook - Add support for namespaced projects - Specify the absolute path to the repo in the message - Notify the service instead of doing the mirroring ourself --- diff --git a/pagure/hooks/files/mirror.py b/pagure/hooks/files/mirror.py index aae03e1..72a63e3 100755 --- a/pagure/hooks/files/mirror.py +++ b/pagure/hooks/files/mirror.py @@ -3,7 +3,10 @@ """Pagure specific hook to mirror a repo to another location. """ +from __future__ import print_function + +import json import os import sys @@ -25,48 +28,29 @@ import pagure.ui.plugins abspath = os.path.abspath(os.environ['GIT_DIR']) -def mirror_repo(): +def main(args): - reponame = pagure.lib.git.get_repo_name(abspath) + repo = pagure.lib.git.get_repo_name(abspath) username = pagure.lib.git.get_username(abspath) + namespace = pagure.lib.git.get_repo_namespace(abspath) if pagure.APP.config.get('HOOK_DEBUG', False): - print 'repo:', reponame, username - - repo = pagure.lib.get_project(pagure.SESSION, reponame, user=username) - if not repo: - print 'Unknown repo %s of username: %s' % (reponame, username) + print('repo:', repo) + print('user:', username) + print('namespace:', namespace) + + project = pagure.lib.get_project( + pagure.SESSION, repo, user=username, namespace=namespace) + if not project: + fullname = reponame + if namespace: + fullname = '%s/%s' % (namespace, reponame) + print('Unknown repo %s of username: %s' % (fullname, username)) sys.exit(1) - plugin = pagure.ui.plugins.get_plugin('Mirroring') - dbobj = plugin.db_object() - - # Get the list of remotes - remotes = [ - remote.strip() - for remote in repo.mirror_hook[0].target.split('\n') - if repo.mirror_hook and remote.strip() - ] - - public_key_name = werkzeug.secure_filename(repo.fullname) - - # Add the remotes - for idx, remote in enumerate(remotes): - lines = pagure.lib.git.read_git_lines( - ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, - '--mirror=push'], abspath) - if pagure.APP.config.get('HOOK_DEBUG', False): - print '\n'.join(lines) - - # Push - for idx, remote in enumerate(remotes): - lines = pagure.lib.git.read_git_lines( - ['push', '%s_%s' % (public_key_name, idx)], abspath) - if pagure.APP.config.get('HOOK_DEBUG', False): - print '\n'.join(lines) - - -def main(args): - mirror_repo() + data = project.to_json(public=True) + data['topic'] = 'pagure.mirror.postcommit' + data['abspath'] = abspath + pagure.lib.REDIS.publish('pagure.mirror', json.dumps(data)) if __name__ == '__main__': From 1cd14bf0531eb937fd2fa064185b0008cf582768 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 23/27] Add a README to pagure-mirror --- diff --git a/pagure-mirror/README.rst b/pagure-mirror/README.rst new file mode 100644 index 0000000..3d7d159 --- /dev/null +++ b/pagure-mirror/README.rst @@ -0,0 +1,11 @@ +Pagure Mirror +============= + +To setup Pagure Mirror for development, it is assumed that all the +dependencies are resolved, then run:: + + PAGURE_CONFIG=/path/to/config PYTHONPATH=. python pagure-mirror/pagure_mirror_server.py + + +Check `doc/usage/pagure_mirror.rst` for further information on how to +set up and configure pagure-mirror for production use. From 99c2e7927b34b41d3162153ac1350e542d74f214 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 24/27] Add systemd unit file for pagure-mirror --- diff --git a/pagure-mirror/pagure_mirror.service b/pagure-mirror/pagure_mirror.service new file mode 100644 index 0000000..daf3ecc --- /dev/null +++ b/pagure-mirror/pagure_mirror.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure Mirror service +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-mirror/pagure_mirror_server.py +Type=simple +User=pagure +Group=pagure +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 0c60a1cd661523f30ce2e01fdb1944fce6b95af8 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 25/27] Use returns instead of sys.exit() The sys.exit() is done later based on the value returned --- diff --git a/pagure/hooks/files/mirror.py b/pagure/hooks/files/mirror.py index 72a63e3..0ff9950 100755 --- a/pagure/hooks/files/mirror.py +++ b/pagure/hooks/files/mirror.py @@ -45,13 +45,15 @@ def main(args): if namespace: fullname = '%s/%s' % (namespace, reponame) print('Unknown repo %s of username: %s' % (fullname, username)) - sys.exit(1) + return 1 data = project.to_json(public=True) data['topic'] = 'pagure.mirror.postcommit' data['abspath'] = abspath pagure.lib.REDIS.publish('pagure.mirror', json.dumps(data)) + return 0 + if __name__ == '__main__': main(sys.argv[1:]) From 026caf315d7a03b5d64c2162a4820dfbe7c0f4c4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 26/27] Rename the pagure-mirror server to pagure_mirror_server as it should be --- diff --git a/pagure-mirror/pagure_ci_server.py b/pagure-mirror/pagure_ci_server.py deleted file mode 100644 index 215112e..0000000 --- a/pagure-mirror/pagure_ci_server.py +++ /dev/null @@ -1,414 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - (c) 2016 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -This server listens to message sent via redis and set-up/remove mirroring -for the corresponding project. - -""" - -import base64 -import json -import logging -import os -import struct - -import requests -import six -import trollius -import trollius_redis -import werkzeug - -from cryptography import utils -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives import serialization - - - -logging.basicConfig(level=logging.DEBUG) -LOG = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure -import pagure.lib - - -CONFIG_TPL = '''host %(name)s - HostName %(host)s - User %(user)s - IdentityFile ~/.ssh/%(keyname)s - -''' - - -# -# Utility methods used to setup/teardown the mirroring -# - - -# Code from: -# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 -def _ssh_write_string(data): - return struct.pack(">I", len(data)) + data - - -def _ssh_write_mpint(value): - data = utils.int_to_bytes(value) - if six.indexbytes(data, 0) & 0x80: - data = b"\x00" + data - return _ssh_write_string(data) - - -# Code from _openssh_public_key_bytes at: -# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 -@trollius.coroutine -def serialize_public_ssh_key( key): - if isinstance(key, rsa.RSAPublicKey): - public_numbers = key.public_numbers() - return b"ssh-rsa " + base64.b64encode( - _ssh_write_string(b"ssh-rsa") + - _ssh_write_mpint(public_numbers.e) + - _ssh_write_mpint(public_numbers.n) - ) - else: - # Since we only write RSA keys, drop the other serializations - return - - -@trollius.coroutine -def create_ssh_key(keyfile): - ''' Create the public and private ssh keys. - - The specified file name will be the private key and the public one will - be in a similar file name ending with a '.pub'. - - ''' - private_key = rsa.generate_private_key( - public_exponent=65537, - key_size=4096, - backend=default_backend() - ) - - private_pem = private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption() - ) - with open(keyfile, 'w') as stream: - stream.write(private_pem) - - public_key = private_key.public_key() - public_pem = yield trollius.From(serialize_public_ssh_key(public_key)) - if public_pem: - with open(keyfile + '.pub', 'w') as stream: - stream.write(public_pem) - - -def split_target(target): - ''' Check if the given target follows the expected model. ''' - LOG.info('Checking target: %s', target) - if target.startswith('http'): - raise PagureException( - 'Invalid target %s, we only support mirroring via ssh' % target) - - if target.startswith('ssh://'): - target = target.replace('ssh://', '', 1) - target = target.replace('/', ':', 1) - - if not '@' in target: - raise PagureException( - 'No user specified in %s, we were expecting it before a `@`' - % target) - if not ':' in target: - raise PagureException( - 'No path specified in %s, we were expecting it after a `:`' - % target) - user, host_path = target.split('@', 1) - host, path = host_path.split(':', 1) - return user, host, path - - -@trollius.coroutine -def check_or_create_ssh_config(ssh_folder, key_name, target): - ''' Check or adjust the ~/.ssh/config file ''' - ssh_config_file = os.path.join(ssh_folder, 'config') - - - for idx, remote in enumerate(target.split('\n')): - remote = remote.strip() - if not remote: - continue - - user, host, path = split_target(remote) - - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path), - 'name': '%s_%s' % (key_name, idx), - 'keyname': key_name, - } - - update = True - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - update = False - - if update: - with open(ssh_config_file, 'a') as stream: - stream.write(ssh_config) - - -@trollius.coroutine -def clean_ssh_config(ssh_folder, key_name, target): - ''' Check or adjust the ~/.ssh/config file ''' - ssh_config_file = os.path.join(ssh_folder, 'config') - - for idx, remote in enumerate(target.split('\n')): - remote = remote.strip() - if not remote: - continue - - user, host, path = split_target(remote) - - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path), - 'name': '%s_%s' % (key_name, idx), - 'keyname': key_name, - } - - data = None - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - data = data.replace(ssh_config, '', 1) - - with open(ssh_config_file, 'w') as stream: - stream.write(data) - - -# -# Actual logic of the service -# - -@trollius.coroutine -def setup_mirroring(project, session, dbobj): - ''' Setup the specified repo for mirroring. - ''' - public_key_name = werkzeug.secure_filename(project.fullname) - ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) - - if not os.path.exists(ssh_folder): - os.makedirs(ssh_folder) - - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - LOG.info('Public key of interest: %s', public_key_file) - - if not os.path.exists(public_key_file): - LOG.info('Creating public key') - yield trollius.From( - create_ssh_key(os.path.join(ssh_folder, public_key_name)) - ) - - LOG.info('Updating ssh configuration') - yield trollius.From( - check_or_create_ssh_config( - ssh_folder, public_key_name, dbobj.target) - ) - - with open(public_key_file) as stream: - public_key = stream.read() - - if dbobj.public_key != public_key: - LOG.info('Updating information in the DB') - dbobj.public_key = public_key - session.add(dbobj) - session.commit() - - -@trollius.coroutine -def mirror_project(repo, session, dbobj, abspath): - ''' Does the actual mirroring of the specified project/repo. - ''' - plugin = pagure.lib.plugins.get_plugin('Mirroring') - dbobj = plugin.db_object() - - # Get the list of remotes - remotes = [ - remote.strip() - for remote in repo.mirror_hook.target.split('\n') - if repo.mirror_hook and remote.strip() - ] - - public_key_name = werkzeug.secure_filename(repo.fullname) - - # Add the remotes - for idx, remote in enumerate(remotes): - lines = pagure.lib.git.read_git_lines( - ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, - '--mirror=push'], abspath) - if pagure.APP.config.get('HOOK_DEBUG', False): - print '\n'.join(lines) - - # Push - for idx, remote in enumerate(remotes): - lines = pagure.lib.git.read_git_lines( - ['push', '%s_%s' % (public_key_name, idx)], abspath) - dbobj.last_log = '\n'.join(lines) - session.add(dbobj) - session.commit() - if pagure.APP.config.get('HOOK_DEBUG', False): - print '\n'.join(lines) - - -@trollius.coroutine -def teardown_mirroring(project, session, dbobj): - ''' Stop the mirroring of the specified repo. - ''' - public_key_name = werkzeug.secure_filename(project.fullname) - ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) - - if not os.path.exists(ssh_folder): - os.makedirs(ssh_folder) - - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - - public_key_name = werkzeug.secure_filename(project.fullname) - private_key_file = os.path.join(ssh_folder, public_key_name) - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - - if os.path.exists(private_key_file): - os.unlink(private_key_file) - - if os.path.exists(public_key_file): - os.unlink(public_key_file) - - yield trollius.From( - clean_ssh_config( - ssh_folder, public_key_name, dbobj.target) - ) - - project.mirror_hook.public_key = None - session.add(project) - session.commit() - - -@trollius.coroutine -def handle_messages(): - ''' Handles connecting to redis and acting upon messages received. - In this case, it means triggering a build on jenkins based on the - information provided. - ''' - - host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') - port = pagure.APP.config.get('REDIS_PORT', 6379) - dbname = pagure.APP.config.get('REDIS_DB', 0) - connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=dbname)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.mirror'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - LOG.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - - reponame = data['name'] - username = data['user']['name'] if data['parent'] else None - namespace = data['namespace'] - abspath = data['abspath'] - LOG.info( - 'Looking for project: %s/%s/%s', namespace, username, reponame) - - session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - repo = pagure.lib.get_project( - session, reponame, - user=username, - namespace=namespace) - if not repo: - print 'Unknown repo %s of username: %s in ns: %s' % ( - reponame, username, namespace) - session.close() - sys.exit(1) - - plugin = pagure.lib.plugins.get_plugin('Mirroring') - dbobj = plugin.db_object() - dbobj = getattr(repo, plugin.backref) - - topic = data.get('topic') - if topic == 'pagure.mirror.postcommit': - yield trollius.From(mirror_project(repo, session, dbobj, abspath)) - elif topic == 'pagure.mirror.setup': - yield trollius.From(setup_mirroring(repo, session, dbobj)) - elif topic == 'pagure.mirror.teardown': - yield trollius.From(teardown_mirroring(repo, session, dbobj)) - else: - LOG.error('Unknown topic found: %s', topic) - - session.close() - LOG.info('Ready for another') - - -def main(): - ''' Start the main async loop. ''' - - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - LOG.info("End Connection") - loop.close() - LOG.info("End") - - -if __name__ == '__main__': - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - LOG.setLevel(logging.DEBUG) - shellhandler = logging.StreamHandler() - shellhandler.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - aslog = logging.getLogger("trollius") - aslog.setLevel(logging.DEBUG) - - shellhandler.setFormatter(formatter) - LOG.addHandler(shellhandler) - main() diff --git a/pagure-mirror/pagure_mirror_server.py b/pagure-mirror/pagure_mirror_server.py new file mode 100644 index 0000000..215112e --- /dev/null +++ b/pagure-mirror/pagure_mirror_server.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and set-up/remove mirroring +for the corresponding project. + +""" + +import base64 +import json +import logging +import os +import struct + +import requests +import six +import trollius +import trollius_redis +import werkzeug + +from cryptography import utils +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + + + +logging.basicConfig(level=logging.DEBUG) +LOG = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +CONFIG_TPL = '''host %(name)s + HostName %(host)s + User %(user)s + IdentityFile ~/.ssh/%(keyname)s + +''' + + +# +# Utility methods used to setup/teardown the mirroring +# + + +# Code from: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 +def _ssh_write_string(data): + return struct.pack(">I", len(data)) + data + + +def _ssh_write_mpint(value): + data = utils.int_to_bytes(value) + if six.indexbytes(data, 0) & 0x80: + data = b"\x00" + data + return _ssh_write_string(data) + + +# Code from _openssh_public_key_bytes at: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 +@trollius.coroutine +def serialize_public_ssh_key( key): + if isinstance(key, rsa.RSAPublicKey): + public_numbers = key.public_numbers() + return b"ssh-rsa " + base64.b64encode( + _ssh_write_string(b"ssh-rsa") + + _ssh_write_mpint(public_numbers.e) + + _ssh_write_mpint(public_numbers.n) + ) + else: + # Since we only write RSA keys, drop the other serializations + return + + +@trollius.coroutine +def create_ssh_key(keyfile): + ''' Create the public and private ssh keys. + + The specified file name will be the private key and the public one will + be in a similar file name ending with a '.pub'. + + ''' + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + with open(keyfile, 'w') as stream: + stream.write(private_pem) + + public_key = private_key.public_key() + public_pem = yield trollius.From(serialize_public_ssh_key(public_key)) + if public_pem: + with open(keyfile + '.pub', 'w') as stream: + stream.write(public_pem) + + +def split_target(target): + ''' Check if the given target follows the expected model. ''' + LOG.info('Checking target: %s', target) + if target.startswith('http'): + raise PagureException( + 'Invalid target %s, we only support mirroring via ssh' % target) + + if target.startswith('ssh://'): + target = target.replace('ssh://', '', 1) + target = target.replace('/', ':', 1) + + if not '@' in target: + raise PagureException( + 'No user specified in %s, we were expecting it before a `@`' + % target) + if not ':' in target: + raise PagureException( + 'No path specified in %s, we were expecting it after a `:`' + % target) + user, host_path = target.split('@', 1) + host, path = host_path.split(':', 1) + return user, host, path + + +@trollius.coroutine +def check_or_create_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + + + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + update = True + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + update = False + + if update: + with open(ssh_config_file, 'a') as stream: + stream.write(ssh_config) + + +@trollius.coroutine +def clean_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + data = None + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + data = data.replace(ssh_config, '', 1) + + with open(ssh_config_file, 'w') as stream: + stream.write(data) + + +# +# Actual logic of the service +# + +@trollius.coroutine +def setup_mirroring(project, session, dbobj): + ''' Setup the specified repo for mirroring. + ''' + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + LOG.info('Public key of interest: %s', public_key_file) + + if not os.path.exists(public_key_file): + LOG.info('Creating public key') + yield trollius.From( + create_ssh_key(os.path.join(ssh_folder, public_key_name)) + ) + + LOG.info('Updating ssh configuration') + yield trollius.From( + check_or_create_ssh_config( + ssh_folder, public_key_name, dbobj.target) + ) + + with open(public_key_file) as stream: + public_key = stream.read() + + if dbobj.public_key != public_key: + LOG.info('Updating information in the DB') + dbobj.public_key = public_key + session.add(dbobj) + session.commit() + + +@trollius.coroutine +def mirror_project(repo, session, dbobj, abspath): + ''' Does the actual mirroring of the specified project/repo. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + + # Get the list of remotes + remotes = [ + remote.strip() + for remote in repo.mirror_hook.target.split('\n') + if repo.mirror_hook and remote.strip() + ] + + public_key_name = werkzeug.secure_filename(repo.fullname) + + # Add the remotes + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, + '--mirror=push'], abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + # Push + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['push', '%s_%s' % (public_key_name, idx)], abspath) + dbobj.last_log = '\n'.join(lines) + session.add(dbobj) + session.commit() + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + +@trollius.coroutine +def teardown_mirroring(project, session, dbobj): + ''' Stop the mirroring of the specified repo. + ''' + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + if os.path.exists(private_key_file): + os.unlink(private_key_file) + + if os.path.exists(public_key_file): + os.unlink(public_key_file) + + yield trollius.From( + clean_ssh_config( + ssh_folder, public_key_name, dbobj.target) + ) + + project.mirror_hook.public_key = None + session.add(project) + session.commit() + + +@trollius.coroutine +def handle_messages(): + ''' Handles connecting to redis and acting upon messages received. + In this case, it means triggering a build on jenkins based on the + information provided. + ''' + + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.mirror'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + LOG.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + reponame = data['name'] + username = data['user']['name'] if data['parent'] else None + namespace = data['namespace'] + abspath = data['abspath'] + LOG.info( + 'Looking for project: %s/%s/%s', namespace, username, reponame) + + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + repo = pagure.lib.get_project( + session, reponame, + user=username, + namespace=namespace) + if not repo: + print 'Unknown repo %s of username: %s in ns: %s' % ( + reponame, username, namespace) + session.close() + sys.exit(1) + + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj = getattr(repo, plugin.backref) + + topic = data.get('topic') + if topic == 'pagure.mirror.postcommit': + yield trollius.From(mirror_project(repo, session, dbobj, abspath)) + elif topic == 'pagure.mirror.setup': + yield trollius.From(setup_mirroring(repo, session, dbobj)) + elif topic == 'pagure.mirror.teardown': + yield trollius.From(teardown_mirroring(repo, session, dbobj)) + else: + LOG.error('Unknown topic found: %s', topic) + + session.close() + LOG.info('Ready for another') + + +def main(): + ''' Start the main async loop. ''' + + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + LOG.info("End Connection") + loop.close() + LOG.info("End") + + +if __name__ == '__main__': + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + LOG.setLevel(logging.DEBUG) + shellhandler = logging.StreamHandler() + shellhandler.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + aslog = logging.getLogger("trollius") + aslog.setLevel(logging.DEBUG) + + shellhandler.setFormatter(formatter) + LOG.addHandler(shellhandler) + main() From 25f972591a3f8979bde4ac9348f6d7092ee15eaf Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: [PATCH 27/27] Start working on the unit-tests for the mirroring server --- diff --git a/tests/test_mirroring_server.py b/tests/test_mirroring_server.py new file mode 100644 index 0000000..45ef3f5 --- /dev/null +++ b/tests/test_mirroring_server.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +Tests for the Pagure mirroring server. + +""" + +# obviously this is fine for testing. +# pylint: disable=locally-disabled, protected-access + +import logging +import os +import shutil +import sys +import tempfile +import unittest + +import mock +import trollius + + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '../pagure-mirror')) + +import pagure # pylint: disable=wrong-import-position +from pagure.exceptions import PagureEvException # pylint: disable=wrong-import-position +import tests # pylint: disable=wrong-import-position +# comes from ev-server/ +import pagure_mirror_server as pms # pylint: disable=wrong-import-position, import-error + +logging.basicConfig(stream=sys.stderr) + + + +class MirroringServerTests(tests.Modeltests): + """Tests for the mirroring server.""" + + def setUp(self): + """Set up the environnment, run before every test.""" + super(MirroringServerTests, self).setUp() + pagure.SESSION = self.session + + # Mock send_email, we never want to send or see emails here. + self.mailpatcher = mock.patch('pagure.lib.notify.send_email') + self.mailpatcher.start() + + # Setup projects + tests.create_projects(self.session) + self.repo = pagure.lib.get_project(self.session, 'test') + self.repo2 = pagure.lib.get_project(self.session, 'test2') + + # Errored out? + self.error = False + + def tearDown(self): + "Stop the patchers, as well as calling super.""" + super(MirroringServerTests, self).tearDown() + self.mailpatcher.stop() + + def _fail_test(self, loop, context): + self.error = True + print(loop) + print(context) + print(self.error) + + def test_setup_mirroring(self): + """Tests for setup_mirroring.""" + abspath = tempfile.mkdtemp(prefix='pagure-tests') + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj = getattr(self.repo, plugin.backref) + + @trollius.coroutine + def _test_setup_mirroring(): + out = yield trollius.From( + pms.setup_mirroring(self.repo, self.session, dbobj)) + self.assertIsNone(out) + self.assertTrue(False) + + loop = trollius.get_event_loop() + loop.set_exception_handler(self._fail_test) + tasks = [ + trollius.async(_test_setup_mirroring()), + ] + loop.run_until_complete(trollius.wait(tasks)) + #print(dir(loop)) + loop.close() + print('end', self.error) + self.assertFalse(self.error) + + shutil.rmtree(abspath) + + +if __name__ == '__main__': + SUITE = unittest.TestLoader().loadTestsFromTestCase( + MirroringServerTests) + unittest.TextTestRunner(verbosity=2).run(SUITE)