From cdb351493ea81d00cd91ae4cc9a8db7b9d3bc71f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 12:23:43 +0000 Subject: [PATCH 1/9] Add a configuration key to disable mirroring in projects Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/new_project.html b/pagure/templates/new_project.html index ed638b7..4be5995 100644 --- a/pagure/templates/new_project.html +++ b/pagure/templates/new_project.html @@ -23,7 +23,9 @@ {% if config.get('PRIVATE_PROJECTS', False) %} {{ render_bootstrap_field(form.private, field_description="To mark the repo private") }} {% endif %} + {% if not config.get('DISABLE_MIRROR_IN', False) %} {{ render_bootstrap_field(form.mirrored_from, field_description="Mirror this project from another git server") }} + {% endif %} {{ render_bootstrap_field(form.create_readme, field_description="Create a README file automatically") }} {% if form.repospanner_region %} {{ render_bootstrap_field(form.repospanner_region, field_description="repoSpanner region to create the project in") }} @@ -66,6 +68,7 @@ $('#private').change(function(){ $('#namespace').removeAttr("disabled"); } }); +{% if not config.get('DISABLE_MIRROR_IN', False) %} function update_if_mirror() { if ($('#mirrored_from').val()){ $('#create_readme').attr("disabled", "disabled"); @@ -80,6 +83,7 @@ $('#mirrored_from').keyup(function(){ update_if_mirror(); }); update_if_mirror(); +{% endif %} {% endblock %} {% endif %} diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 0a16d69..374fd2b 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -1042,6 +1042,12 @@ def new_project(): ignore_existing_repos = False mirrored_from = form.mirrored_from.data + if mirrored_from and pagure_config.get("DISABLE_MIRROR_IN", False): + flask.flash( + "Mirroring in projects has been disabled in this instance", + "error", + ) + return flask.render_template("new_project.html", form=form) try: task = pagure.lib.query.new_project( diff --git a/tests/test_pagure_flask_ui_app.py b/tests/test_pagure_flask_ui_app.py index f9abf00..f3d9924 100644 --- a/tests/test_pagure_flask_ui_app.py +++ b/tests/test_pagure_flask_ui_app.py @@ -340,6 +340,10 @@ class PagureFlaskApptests(tests.Modeltests): with tests.user_set(self.app.application, user): output = self.app.get('/new/') self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + '', output_text) csrf_token = self.get_csrf(output=output) @@ -360,6 +364,39 @@ class PagureFlaskApptests(tests.Modeltests): '

This repo is brand new and meant to be mirrored from ' 'https://example.com/foo/bar.git !

', output_text) + @patch.dict('pagure.config.config', {'DISABLE_MIRROR_IN': True}) + def test_new_project_mirrored_mirror_disabled(self): + """ Test the new_project with a mirrored repo when that feature is + disabled. + """ + + user = tests.FakeUser(username='foo') + with tests.user_set(self.app.application, user): + output = self.app.get('/new/') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertNotIn( + '', output_text) + + csrf_token = self.get_csrf(output=output) + + data = { + 'description': 'Project #1', + 'name': 'project-1', + 'mirrored_from': 'https://example.com/foo/bar.git', + 'csrf_token': csrf_token, + } + + output = self.app.post('/new/', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'New project - Pagure', output_text) + self.assertIn( + ' Mirroring in projects has been disabled in ' + 'this instance', output_text) + def test_new_project(self): """ Test the new_project endpoint. """ # Before From 08ed05a86c4c419c8579cc15e6967ee754cd101e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 12:23:43 +0000 Subject: [PATCH 2/9] Add a page in the settings showing the last log of the last mirror run Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/settings.html b/pagure/templates/settings.html index ce0c691..4d4a3b3 100644 --- a/pagure/templates/settings.html +++ b/pagure/templates/settings.html @@ -72,6 +72,13 @@ href="#quickreplies-tab" role="tab" aria-controls="quickreplies">Quick Replies {% endif %} + {% if not config.get('DISABLE_MIRROR_IN', False) + and (repo.user.user == g.fas_user.username or pagure_admin) + and repo.mirrored_from %} + Mirror log + {% endif %} + Regenerate Repos @@ -761,7 +768,7 @@ {% else %} - {% endif %} @@ -976,6 +983,14 @@ {% endif %} + {% if not config.get('DISABLE_MIRROR_IN', False) + and (repo.user.user == g.fas_user.username or pagure_admin) + and repo.mirrored_from %} +
+ {% include 'settings_mirrorlog.html' %} +
+ {% endif %} +

Regenerate Repos diff --git a/pagure/templates/settings_mirrorlog.html b/pagure/templates/settings_mirrorlog.html new file mode 100644 index 0000000..6cc0dad --- /dev/null +++ b/pagure/templates/settings_mirrorlog.html @@ -0,0 +1,11 @@ + +

+ Mirrorring in log +

+

+ Here below is the log of the last attempt to mirror in this project. +

+ +
+  {{ repo.mirrored_from_last_log }}
+
From d114c702f99bbc2efde186920e8d054d55b68292 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 12:23:43 +0000 Subject: [PATCH 3/9] Fix ensuring the list of branches on the new PR page matches the repo Before this commit, the list of branches shown in the drop-down for the branch from in the new PR page was the list of branches of the target repo instead of being the list of branches for the destination repo. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 68845da..d26dd66 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -1760,7 +1760,7 @@ def new_request_pull( repo=repo, username=username, orig_repo=orig_repo, - parent_branches=sorted(orig_repo.listall_branches()), + parent_branches=sorted(flask.g.repo_obj.listall_branches()), diff_commits=diff_commits, diff=diff, form=form, diff --git a/tests/test_pagure_flask_ui_fork.py b/tests/test_pagure_flask_ui_fork.py index 311da94..3ac12ab 100644 --- a/tests/test_pagure_flask_ui_fork.py +++ b/tests/test_pagure_flask_ui_fork.py @@ -2280,7 +2280,8 @@ index 0000000..2a552bb os.path.join(self.path, 'requests'), bare=True) repo = pagure.lib.query.get_authorized_project(self.session, 'test') - fork = pagure.lib.query.get_authorized_project(self.session, 'test', user='foo') + fork = pagure.lib.query.get_authorized_project( + self.session, 'test', user='foo') set_up_git_repo( self.session, self.path, new_project=fork, @@ -2339,6 +2340,11 @@ index 0000000..2a552bb placeholder="Describe your changes" tabindex=1> More information
''', output_text) + self.assertIn( + ' master', + output_text) csrf_token = self.get_csrf(output=output) @@ -2553,6 +2559,86 @@ More information self.assertIsNotNone(request.commit_stop) @patch('pagure.lib.notify.send_email') + def test_new_request_pull_from_fork_branch(self, send_email): + """ Test creating a fork to fork PR. """ + send_email.return_value = True + + # Create main repo with some content + tests.create_projects(self.session) + tests.create_projects_git( + os.path.join(self.path, "repos"), + bare=True + ) + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git")) + + # Create fork repo with more content + tests.create_projects( + self.session, + is_fork=True, + hook_token_suffix='fork') + tests.create_projects_git( + os.path.join(self.path, "repos", "forks", "pingou"), + bare=True + ) + tests.add_content_git_repo( + os.path.join(self.path, "repos", "forks", "pingou", "test.git")) + tests.add_readme_git_repo( + os.path.join(self.path, "repos", "forks", "pingou", "test.git"), + branch='feature') + tests.add_readme_git_repo( + os.path.join(self.path, "repos", "forks", "pingou", "test.git"), + branch='random_branch') + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + data = { + 'csrf_token': self.get_csrf(), + } + + output = self.app.post( + '/do_fork/test', data=data, + follow_redirects=True) + self.assertEqual(output.status_code, 200) + + # Check that Ralph's fork do exist + output = self.app.get('/fork/pingou/test') + self.assertEqual(output.status_code, 200) + + tests.create_projects_git( + os.path.join(self.path, 'requests'), bare=True) + + fork = pagure.lib.query.get_authorized_project( + self.session, 'test', user='ralph') + + set_up_git_repo( + self.session, self.path, new_project=fork, + branch_from='feature', mtype='FF') + + # Try opening a pull-request + output = self.app.get( + '/fork/pingou/test/diff/master..feature') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Create new Pull Request for master - ' + 'fork/pingou/test\n - Pagure', output_text) + self.assertIn( + '\n', + output_text) + self.assertIn( + ' master', + output_text) + self.assertIn( + ' random_branch', + output_text) + + + @patch('pagure.lib.notify.send_email') def test_new_request_pull_fork_to_fork_pr_disabled(self, send_email): """ Test creating a fork to fork PR. """ send_email.return_value = True From d28b70be9bf927c61b5253a3f6d76f1db5d314f7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 12:23:43 +0000 Subject: [PATCH 4/9] Fix underscore title in the API doc Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/api/fork.py b/pagure/api/fork.py index d64059a..ef5ef4b 100644 --- a/pagure/api/fork.py +++ b/pagure/api/fork.py @@ -522,7 +522,7 @@ def api_pull_request_merge(repo, requestid, username=None, namespace=None): def api_pull_request_rebase(repo, requestid, username=None, namespace=None): """ Rebase a pull-request - -------------------- + --------------------- Instruct Pagure to rebase a pull request. This is an asynchronous call. From 2a925db5fe24704c419ac146ba1324a5ccc27fe3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 13:25:14 +0000 Subject: [PATCH 5/9] Move the SMTP configuration keys into their own section Signed-off-by: Pierre-Yves Chibon --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 5e23710..04e4e92 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -761,9 +761,11 @@ the default branch of the repo will be the default target of all pull requests. Defaults to: ``False``. +SMTP configuration +~~~~~~~~~~~~~~~~~~ SMTP_SERVER -~~~~~~~~~~~ +^^^^^^^^^^^ This configuration key specifies the SMTP server to use when sending emails. @@ -772,7 +774,7 @@ Defaults to: ``localhost``. SMTP_PORT -~~~~~~~~~ +^^^^^^^^^ This configuration key specifies the SMTP server port. @@ -785,7 +787,7 @@ Defaults to: ``25`` SMTP_SSL -~~~~~~~~ +^^^^^^^^ This configuration key specifies whether the SMTP connections should be secured over SSL. @@ -794,7 +796,7 @@ Defaults to: ``False`` SMTP_USERNAME -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ This configuration key allows usage of SMTP with auth. @@ -804,7 +806,7 @@ Defaults to: ``None`` SMTP_PASSWORD -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ This configuration key allows usage of SMTP with auth. From d85017c7170d5db4380c0ab7bfd4522465e75056 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 13:42:49 +0000 Subject: [PATCH 6/9] Document the DISABLE_MIRROR_IN configuration key Signed-off-by: Pierre-Yves Chibon --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 04e4e92..d02c0cd 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1582,6 +1582,15 @@ pushed to the default branch. Defaults to: ``False`` +DISABLE_MIRROR_IN +~~~~~~~~~~~~~~~~~ + +This configuration key allows a pagure instance to not support mirroring in +projects (from third party git server). + +Defaults to: ``False`` + + RepoSpanner Options ------------------- From c0512fc3bd9882221994fad13c68e2d18ed4d7aa Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 13:43:13 +0000 Subject: [PATCH 7/9] Document the MQTT configuration keys Signed-off-by: Pierre-Yves Chibon --- diff --git a/doc/configuration.rst b/doc/configuration.rst index d02c0cd..076deca 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1736,6 +1736,107 @@ SSH_COMMAND_NON_REPOSPANNER The command to run if a repository is not on repospanner when aclchecker is in use. +MQTT Options +------------ + +If approprietly configured pagure supports sending messages to an MQTT +message queue. + +Here below are the different configuration options to make it so. + +MQTT_NOTIFICATIONS +~~~~~~~~~~~~~~~~~~ + +Global configuration key to turn on or off the code to send notifications +to an MQTT message queue. + +Defaults to: ``False`` + +MQTT_HOST +~~~~~~~~~ + +Host name of the MQTT server to send the MQTT notifications to. + +Defaults to: ``None`` + +MQTT_PORT +~~~~~~~~~ + +Port of the MQTT server to use to send the MQTT notifications to. + +Defaults to: ``None`` + + +MQTT_USERNAME +~~~~~~~~~~~~~ + +Username to authenticate to the MQTT server as. + +Defaults to: ``None`` + + +MQTT_PASSWORD +~~~~~~~~~~~~~ + +Password to authenticate to the MQTT server with. + +Defaults to: ``None`` + + +MQTT_CA_CERTS +~~~~~~~~~~~~~ + +When using SSL-based authentication to the MQTT server, use this +configuration key to point to the CA cert to use. + +Defaults to: ``None`` + + +MQTT_CERTFILE +~~~~~~~~~~~~~ + +When using SSL-based authentication to the MQTT server, use this +configuration key to point to the cert file to use. + +Defaults to: ``None`` + + +MQTT_KEYFILE +~~~~~~~~~~~~~ + +When using SSL-based authentication to the MQTT server, use this +configuration key to point to the key file to use. + +Defaults to: ``None`` + + +MQTT_CERT_REQS +~~~~~~~~~~~~~~ + +When using SSL-based authentication to the MQTT server, use this +configuration key to specify if the CERT is required. + +Defaults to: ``ssl.CERT_REQUIRED`` (from python's ssl library) + + +MQTT_TLS_VERSION +~~~~~~~~~~~~~~~~ + +When using SSL-based authentication to the MQTT server, use this +configuration key to specify the TLS protocols to support/use. + +Defaults to: ``ssl.PROTOCOL_TLS`` (from python's ssl library) + + +MQTT_CIPHERS +~~~~~~~~~~~~ + +When using SSL-based authentication to the MQTT server, use this +configuration key to specify the ciphers. + +Defaults to: ``None`` + + Deprecated configuration keys ----------------------------- diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index d82dc82..d14d7bf 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -109,7 +109,7 @@ def blinker_publish(topic, message): def mqtt_publish(topic, message): """ Try to publish a message on a MQTT message bus. """ - if not pagure_config.get("MQTT_NOTIFICATIONS", True): + if pagure_config.get("MQTT_NOTIFICATIONS", False): return mqtt_host = pagure_config.get("MQTT_HOST") From 420b95b8d735bbc8903dda773a60be3f59550ace Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 17 2018 17:16:56 +0000 Subject: [PATCH 8/9] Move the default MQTT protocol to be TLS v1.2 ssl.PROTOCOL_TLS has been deprecated in python and 1.2 is the latest version to date. Signed-off-by: Pierre-Yves Chibon --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 076deca..0b04c30 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1825,7 +1825,7 @@ MQTT_TLS_VERSION When using SSL-based authentication to the MQTT server, use this configuration key to specify the TLS protocols to support/use. -Defaults to: ``ssl.PROTOCOL_TLS`` (from python's ssl library) +Defaults to: ``ssl.PROTOCOL_TLSv1_2`` (from python's ssl library) MQTT_CIPHERS diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index d14d7bf..d1e9bb0 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -122,7 +122,9 @@ def mqtt_publish(topic, message): mqtt_certfile = pagure_config.get("MQTT_CERTFILE") mqtt_keyfile = pagure_config.get("MQTT_KEYFILE") mqtt_cert_reqs = pagure_config.get("MQTT_CERT_REQS", ssl.CERT_REQUIRED) - mqtt_tls_version = pagure_config.get("MQTT_TLS_VERSION", ssl.PROTOCOL_TLS) + mqtt_tls_version = pagure_config.get( + "MQTT_TLS_VERSION", ssl.PROTOCOL_TLSv1_2 + ) mqtt_ciphers = pagure_config.get("MQTT_CIPHERS") # We catch Exception if we want :-p From 71aa91a17fda9202b5d8aa253aec60a089f98274 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 18 2018 10:43:58 +0000 Subject: [PATCH 9/9] Make the filepath shown in the UI an unicode Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/repo_pull_request.html b/pagure/templates/repo_pull_request.html index bca3ccc..689d357 100644 --- a/pagure/templates/repo_pull_request.html +++ b/pagure/templates/repo_pull_request.html @@ -678,7 +678,7 @@
file added
- {{filepath}} + {{ filepath | unicode }}
{%- endmacro %}