From 8f49484e5e6f43d3db893d5a274b62fc2c12d9d5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 06 2016 13:55:25 +0000 Subject: [PATCH 1/10] Introduce a default hook/plugin This hook/plugin is meant to be activated for all projects on pagure and for the moment, all it does is resetting the merge status of all open pull-requests upon push to the main branch. --- diff --git a/pagure/hooks/default.py b/pagure/hooks/default.py new file mode 100644 index 0000000..97f92bb --- /dev/null +++ b/pagure/hooks/default.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +import sqlalchemy as sa +import wtforms +try: + from flask_wtf import FlaskForm +except ImportError: + from flask_wtf import Form as FlaskForm +from sqlalchemy.orm import relation +from sqlalchemy.orm import backref + +import pagure.lib +from pagure.hooks import BaseHook, RequiredIf +from pagure.lib.model import BASE, Project +from pagure import SESSION, APP, get_repo_path + + +class DefaultTable(BASE): + """ Stores information about the CI linked to on a project. + + Table -- hook_default + """ + + __tablename__ = 'hook_default' + + id = sa.Column(sa.Integer, primary_key=True) + project_id = sa.Column( + sa.Integer, + sa.ForeignKey( + 'projects.id', onupdate='CASCADE', ondelete='CASCADE'), + nullable=False, + unique=True, + index=True) + active = sa.Column(sa.Boolean, nullable=False, default=False) + + project = relation( + 'Project', remote_side=[Project.id], + backref=backref( + 'default_hook', cascade="delete, delete-orphan", + single_parent=True, uselist=False) + ) + + +class DefaultForm(FlaskForm): + ''' Form to configure the default hook. ''' + active = wtforms.BooleanField( + 'Active', + [wtforms.validators.Optional()] + ) + + def __init__(self, *args, **kwargs): + """ Calls the default constructor with the normal argument but + uses the list of collection provided to fill the choices of the + drop-down list. + """ + super(DefaultForm, self).__init__(*args, **kwargs) + + +class Default(BaseHook): + ''' Mail hooks. ''' + + name = 'default' + description = 'Default hooks that should be enabled for each and '\ + 'every project.' + + form = DefaultForm + db_object = DefaultTable + backref = 'default_hook' + form_fields = ['active'] + + @classmethod + def install(cls, project, dbobj): + ''' Method called to install the hook for a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + repopaths = [get_repo_path(project)] + + cls.base_install(repopaths, dbobj, 'default', 'default_hook.py') + + @classmethod + def remove(cls, project): + ''' Method called to remove the hook of a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + repopaths = [get_repo_path(project)] + + cls.base_remove(repopaths, 'default') diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py new file mode 100644 index 0000000..7526443 --- /dev/null +++ b/pagure/hooks/files/default_hook.py @@ -0,0 +1,84 @@ +#! /usr/bin/env python2 + + +"""Pagure specific hook to be added to all projects in pagure by default. +""" + +import os +import sys + +import pygit2 + +from sqlalchemy.exc import SQLAlchemyError + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.exceptions +import pagure.lib.link + + +abspath = os.path.abspath(os.environ['GIT_DIR']) + + +def run_as_post_receive_hook(): + + for line in sys.stdin: + if pagure.APP.config.get('HOOK_DEBUG', False): + print line + (oldrev, newrev, refname) = line.strip().split(' ', 2) + + if pagure.APP.config.get('HOOK_DEBUG', False): + print ' -- Old rev' + print oldrev + print ' -- New rev' + print newrev + print ' -- Ref name' + print refname + + # Retrieve the default branch + repo_obj = pygit2.Repository(abspath) + default_branch = None + if not repo_obj.is_empty and not repo_obj.head_is_unborn: + default_branch = repo_obj.head.shorthand + + # Skip all branch but the default one + refname = refname.replace('refs/heads/', '') + if refname != default_branch: + continue + + if set(newrev) == set(['0']): + print "Deleting a reference/branch, so we won't run the "\ + "pagure hook" + return + + repo = pagure.lib.git.get_repo_name(abspath) + username = pagure.lib.git.get_username(abspath) + namespace = pagure.lib.git.get_repo_namespace(abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print 'repo:', repo + print 'user:', username + print 'namespace:', namespace + + project = pagure.lib.get_project( + pagure.SESSION, repo, user=username, namespace=namespace) + try: + # Reset the merge_status of all opened PR to refresh their cache + pagure.lib.reset_status_pull_request(pagure.SESSION, project) + pagure.SESSION.commit() + except SQLAlchemyError as err: # pragma: no cover + pagure.SESSION.rollback() + pagure.APP.logger.exception(err) + print 'An error occured while running the default hook, please '\ + 'report it to an admin.' + + +def main(args): + run_as_post_receive_hook() + + +if __name__ == '__main__': + main(sys.argv[1:]) From 8d1854ded01b4ece0409e802393e8a36e5141ddd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 06 2016 13:56:28 +0000 Subject: [PATCH 2/10] Do not include the default hook in the list of plugins --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 1e88233..0f0dc63 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -38,6 +38,9 @@ def get_plugin_names(blacklist=None): for plugin in plugins if plugin.name not in blacklist ] + # The default hook is not one we show + if 'default' in output: + output.remove('default') return sorted(output) From 295cfb757b3acfa4135ecd0788fedbe451aa5ad6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 06 2016 15:09:16 +0000 Subject: [PATCH 3/10] Move the logic related to plugins to pagure.lib.plugins and adjust calls Adjust the unit-tests while at it to cover this change --- diff --git a/pagure/lib/model.py b/pagure/lib/model.py index 7edf9bd..f639e14 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -68,7 +68,7 @@ def create_tables(db_url, alembic_ini=None, acls=None, debug=False): else: engine = create_engine(db_url, echo=debug, client_encoding='utf8') - from pagure.ui.plugins import get_plugin_tables + from pagure.lib.plugins import get_plugin_tables get_plugin_tables() BASE.metadata.create_all(engine) # engine.execute(collection_package_create_view(driver=engine.driver)) diff --git a/pagure/lib/plugins.py b/pagure/lib/plugins.py new file mode 100644 index 0000000..84c281c --- /dev/null +++ b/pagure/lib/plugins.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from straight.plugin import load + +from pagure.lib.model import BASE + + +def get_plugin_names(blacklist=None): + ''' Return the list of plugins names. ''' + from pagure.hooks import BaseHook + plugins = load('pagure.hooks', subclasses=BaseHook) + if not blacklist: + blacklist = [] + elif not isinstance(blacklist, list): + blacklist = [blacklist] + + output = [ + plugin.name + for plugin in plugins + if plugin.name not in blacklist + ] + # The default hook is not one we show + if 'default' in output: + output.remove('default') + return sorted(output) + + +def get_plugin_tables(): + ''' Return the list of all plugins. ''' + plugins = load('pagure.hooks', subclasses=BASE) + return plugins + + +def get_plugin(plugin_name): + ''' Return the list of plugins names. ''' + from pagure.hooks import BaseHook + plugins = load('pagure.hooks', subclasses=BaseHook) + for plugin in plugins: + if plugin.name == plugin_name: + return plugin diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 0f0dc63..b6bf3b2 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -14,50 +14,15 @@ import flask from sqlalchemy.exc import SQLAlchemyError -from straight.plugin import load -from pagure.hooks import BaseHook import pagure.exceptions import pagure.lib +import pagure.lib.plugins import pagure.forms from pagure import APP, SESSION, login_required -from pagure.lib.model import BASE from pagure.exceptions import FileNotFoundException -def get_plugin_names(blacklist=None): - ''' Return the list of plugins names. ''' - plugins = load('pagure.hooks', subclasses=BaseHook) - if not blacklist: - blacklist = [] - elif not isinstance(blacklist, list): - blacklist = [blacklist] - - output = [ - plugin.name - for plugin in plugins - if plugin.name not in blacklist - ] - # The default hook is not one we show - if 'default' in output: - output.remove('default') - return sorted(output) - - -def get_plugin_tables(): - ''' Return the list of all plugins. ''' - plugins = load('pagure.hooks', subclasses=BASE) - return plugins - - -def get_plugin(plugin_name): - ''' Return the list of plugins names. ''' - plugins = load('pagure.hooks', subclasses=BaseHook) - for plugin in plugins: - if plugin.name == plugin_name: - return plugin - - @APP.route('//settings//', methods=('GET', 'POST')) @APP.route('//settings/', methods=('GET', 'POST')) @APP.route('///settings//', methods=('GET', 'POST')) @@ -112,7 +77,7 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): if plugin in APP.config.get('DISABLED_PLUGINS', []): flask.abort(404, 'Plugin disabled') - plugin = get_plugin(plugin) + plugin = pagure.lib.plugins.get_plugin(plugin) fields = [] new = True dbobj = plugin.db_object() diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index c857a4a..f68584a 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -44,6 +44,7 @@ from binaryornot.helpers import is_binary_string import pagure.exceptions import pagure.lib import pagure.lib.git +import pagure.lib.plugins import pagure.forms import pagure import pagure.ui.plugins @@ -939,7 +940,7 @@ def view_settings(repo, username=None, namespace=None): 403, 'You are not allowed to change the settings for this project') - plugins = pagure.ui.plugins.get_plugin_names( + plugins = pagure.lib.plugins.get_plugin_names( APP.config.get('DISABLED_PLUGINS')) tags = pagure.lib.get_tags_of_project(SESSION, repo) diff --git a/tests/test_pagure_flask_ui_plugins.py b/tests/test_pagure_flask_ui_plugins.py index 4c2e2aa..4546bf1 100644 --- a/tests/test_pagure_flask_ui_plugins.py +++ b/tests/test_pagure_flask_ui_plugins.py @@ -25,8 +25,8 @@ sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) import pagure.lib +import pagure.lib.plugins import pagure.hooks -import pagure.ui.plugins import tests @@ -66,7 +66,7 @@ class PagureFlaskPluginstests(tests.Modeltests): def test_get_plugin_names(self): """ Test the get_plugin_names function. """ - names = pagure.ui.plugins.get_plugin_names() + names = pagure.lib.plugins.get_plugin_names() self.assertEqual( sorted(names), [ @@ -78,7 +78,7 @@ class PagureFlaskPluginstests(tests.Modeltests): def test_get_plugin(self): """ Test the get_plugin function. """ - name = pagure.ui.plugins.get_plugin('Mail') + name = pagure.lib.plugins.get_plugin('Mail') self.assertEqual(str(name), "") def test_view_plugin_page(self): From afcf11a0ac792a93d11cefcf6cd9f88df6a0ead4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 06 2016 15:10:06 +0000 Subject: [PATCH 4/10] Install the default hook/plugin when creating a new project Adjust the unit-tests for this change --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index b4f55bb..c5d0b70 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -46,6 +46,7 @@ import pagure.exceptions import pagure.lib.git import pagure.lib.login import pagure.lib.notify +import pagure.lib.plugins import pagure.pfmarkdown from pagure.lib import model @@ -1084,6 +1085,7 @@ def new_project(session, user, name, blacklist, allowed_prefix, # Make sure we won't have SQLAlchemy error before we create the repo session.flush() + # Add the readme file if it was asked if not add_readme: pygit2.init_repository(gitrepo, bare=True) else: @@ -1107,6 +1109,7 @@ def new_project(session, user, name, blacklist, allowed_prefix, pygit2.clone_repository(temp_gitrepo_path, gitrepo, bare=True) shutil.rmtree(temp_gitrepo_path) + # Make the repo exportable via apache http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') if not os.path.exists(http_clone_file): with open(http_clone_file, 'w') as stream: @@ -1143,6 +1146,16 @@ def new_project(session, user, name, blacklist, allowed_prefix, requestrepo, bare=True, mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + # Install the default hook + plugin = pagure.lib.plugins.get_plugin('default') + dbobj = plugin.db_object() + dbobj.active = True + dbobj.project_id = project.id + session.add(dbobj) + session.flush() + plugin.set_up(project) + plugin.install(project, dbobj) + # create the project in the db session.commit() diff --git a/tests/test_pagure_lib.py b/tests/test_pagure_lib.py index cb2d7ba..a34c77f 100644 --- a/tests/test_pagure_lib.py +++ b/tests/test_pagure_lib.py @@ -764,6 +764,7 @@ class PagureLibtests(tests.Modeltests): ) # Create a new project + pagure.APP.config['GIT_FOLDER'] = gitfolder msg = pagure.lib.new_project( session=self.session, user='pingou', @@ -1140,6 +1141,7 @@ class PagureLibtests(tests.Modeltests): docfolder = os.path.join(self.path, 'docs') ticketfolder = os.path.join(self.path, 'tickets') requestfolder = os.path.join(self.path, 'requests') + pagure.APP.config['GIT_FOLDER'] = gitfolder projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) From b803bd95381c528eb620b7091c966d3c528fd42d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 06 2016 19:09:36 +0000 Subject: [PATCH 5/10] We cannot strip None objects --- diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 255dbca..85fb951 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -163,7 +163,7 @@ def update_issue(repo, issueid, username=None, namespace=None): assignee = form.assignee.data.strip() or None new_status = form.status.data.strip() or None - close_status = form.close_status.data.strip() or None + close_status = form.close_status.data or None if new_status != 'Closed': close_status = None if close_status not in repo.close_status: From b90d552acd9b71b1a7ed91a5ea1d599473ff40ee Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 07 2016 07:11:54 +0000 Subject: [PATCH 6/10] Style fix (tab -> spaces) --- diff --git a/pagure/hooks/default.py b/pagure/hooks/default.py index 97f92bb..556f8cf 100644 --- a/pagure/hooks/default.py +++ b/pagure/hooks/default.py @@ -69,7 +69,7 @@ class Default(BaseHook): name = 'default' description = 'Default hooks that should be enabled for each and '\ - 'every project.' + 'every project.' form = DefaultForm db_object = DefaultTable From 149eaf781d7f881b92f3d4855c29118723c97596 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 07 2016 07:12:03 +0000 Subject: [PATCH 7/10] Do not allow changing the default plugin via the UI --- diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index b6bf3b2..84e0117 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -77,6 +77,9 @@ def view_plugin(repo, plugin, username=None, namespace=None, full=True): if plugin in APP.config.get('DISABLED_PLUGINS', []): flask.abort(404, 'Plugin disabled') + if plugin == 'default': + flask.abort(403, 'This plugin cannot be changed') + plugin = pagure.lib.plugins.get_plugin(plugin) fields = [] new = True From 317b758c2c5812f43a49315a5e6cb5816b38f70f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 07 2016 07:12:19 +0000 Subject: [PATCH 8/10] Add unit-tests for the default hook/plugin --- diff --git a/tests/test_pagure_flask_ui_plugins_default_hook.py b/tests/test_pagure_flask_ui_plugins_default_hook.py new file mode 100644 index 0000000..017c472 --- /dev/null +++ b/tests/test_pagure_flask_ui_plugins_default_hook.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +__requires__ = ['SQLAlchemy >= 0.8'] +import pkg_resources + +import json +import unittest +import shutil +import sys +import os + +import pygit2 +from mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure.lib +import tests + + +class PagureFlaskPluginDefaultHooktests(tests.Modeltests): + """ Tests for default_hook plugin of pagure """ + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureFlaskPluginDefaultHooktests, self).setUp() + + pagure.APP.config['TESTING'] = True + pagure.SESSION = self.session + pagure.ui.SESSION = self.session + pagure.ui.app.SESSION = self.session + pagure.ui.plugins.SESSION = self.session + pagure.ui.repo.SESSION = self.session + pagure.ui.filters.SESSION = self.session + + pagure.APP.config['GIT_FOLDER'] = self.path + pagure.APP.config['TICKETS_FOLDER'] = os.path.join( + self.path, 'tickets') + pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( + self.path, 'requests') + pagure.APP.config['DOCS_FOLDER'] = os.path.join( + self.path, 'docs') + self.app = pagure.APP.test_client() + + def test_plugin_default_ui(self): + """ Test the default hook plugin on/off endpoint. """ + + tests.create_projects(self.session) + tests.create_projects_git(self.path) + + user = tests.FakeUser(username='pingou') + with tests.user_set(pagure.APP, user): + output = self.app.get('/test/settings/default') + self.assertEqual(output.status_code, 403) + + def test_plugin_default_install(self): + """ Check that the default plugin is correctly installed when a + project is created. + """ + + msg = pagure.lib.new_project( + self.session, + user='pingou', + name='test', + blacklist=[], + allowed_prefix=[], + gitfolder=self.path, + docfolder=os.path.join(self.path, 'docs'), + ticketfolder=os.path.join(self.path, 'tickets'), + requestfolder=os.path.join(self.path, 'requests'), + description=None, + url=None, avatar_email=None, + parent_id=None, + add_readme=False, + userobj=None, + prevent_40_chars=False, + namespace=None + ) + + self.assertEqual(msg, 'Project "test" created') + + self.assertTrue(os.path.exists(os.path.join( + self.path, 'test.git', 'hooks', 'post-receive.default'))) + self.assertTrue(os.path.exists(os.path.join( + self.path, 'test.git', 'hooks', 'post-receive'))) + + +if __name__ == '__main__': + SUITE = unittest.TestLoader().loadTestsFromTestCase( + PagureFlaskPluginDefaultHooktests) + unittest.TextTestRunner(verbosity=2).run(SUITE) From d9ec661c985ab0e1388bc6740247d23f2939fd96 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 07 2016 07:18:45 +0000 Subject: [PATCH 9/10] Some more style fixes (tab -> spaces and indentation) --- diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index 7526443..fd550a3 100644 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -55,17 +55,17 @@ def run_as_post_receive_hook(): "pagure hook" return - repo = pagure.lib.git.get_repo_name(abspath) - username = pagure.lib.git.get_username(abspath) - namespace = pagure.lib.git.get_repo_namespace(abspath) + repo = pagure.lib.git.get_repo_name(abspath) + username = pagure.lib.git.get_username(abspath) + namespace = pagure.lib.git.get_repo_namespace(abspath) if pagure.APP.config.get('HOOK_DEBUG', False): print 'repo:', repo print 'user:', username print 'namespace:', namespace project = pagure.lib.get_project( - pagure.SESSION, repo, user=username, namespace=namespace) - try: + pagure.SESSION, repo, user=username, namespace=namespace) + try: # Reset the merge_status of all opened PR to refresh their cache pagure.lib.reset_status_pull_request(pagure.SESSION, project) pagure.SESSION.commit() @@ -73,11 +73,11 @@ def run_as_post_receive_hook(): pagure.SESSION.rollback() pagure.APP.logger.exception(err) print 'An error occured while running the default hook, please '\ - 'report it to an admin.' + 'report it to an admin.' def main(args): - run_as_post_receive_hook() + run_as_post_receive_hook() if __name__ == '__main__': diff --git a/pagure/hooks/files/pagure_hook.py b/pagure/hooks/files/pagure_hook.py index b56071a..74495b1 100755 --- a/pagure/hooks/files/pagure_hook.py +++ b/pagure/hooks/files/pagure_hook.py @@ -196,7 +196,7 @@ def run_as_post_receive_hook(): def main(args): - run_as_post_receive_hook() + run_as_post_receive_hook() if __name__ == '__main__': From 3cc8fbc6a4c30acb42a395104f262194d40cbf92 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 07 2016 15:05:53 +0000 Subject: [PATCH 10/10] Fix docstrings in the default hook --- diff --git a/pagure/hooks/default.py b/pagure/hooks/default.py index 556f8cf..5d34a69 100644 --- a/pagure/hooks/default.py +++ b/pagure/hooks/default.py @@ -24,7 +24,7 @@ from pagure import SESSION, APP, get_repo_path class DefaultTable(BASE): - """ Stores information about the CI linked to on a project. + """ Stores information about the default hook of a project. Table -- hook_default """ @@ -65,7 +65,7 @@ class DefaultForm(FlaskForm): class Default(BaseHook): - ''' Mail hooks. ''' + ''' Default hooks. ''' name = 'default' description = 'Default hooks that should be enabled for each and '\