From 828527dae5804e7dc736a25721272fcf18b03339 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 1/7] Fix the last modified date on the PR list last_updated is changed automatically every time this row of the table is updated (for example when the cached merged status is cleared). updated_on is manually updated when there was a changed made to the PR that is of interest for the user Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/_render_pullrequests.html b/pagure/templates/_render_pullrequests.html index 3d4a3e5..f886ce2 100644 --- a/pagure/templates/_render_pullrequests.html +++ b/pagure/templates/_render_pullrequests.html @@ -93,8 +93,8 @@ }}">{{ request.date_created | humanize}} by {{ request.user.user }}. Modified {{ request.last_updated | humanize}} + request.updated_on | format_datetime + }}">{{ request.updated_on | humanize}} {% endif %} {% if showproject %} From 4840292211271a8d6477122279d900ced4ccc2f9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 2/7] Have a dedicated variable to bypass gitolite when doing a git push This variable is then checked in gitolite's update hook to avoid triggering it which would fail the push since we're pushing directly to the git repo on disk w/o going through gitolite itself. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 08b3f80..893fc69 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -992,6 +992,7 @@ class TemporaryClone(object): _log.debug("Opts: %s", opts) env = os.environ.copy() env["GL_USER"] = username + env["BYPASS_GITOLITE"] = "defined" env.update(extra) out = subprocess.check_output( ["git"] + opts + ["push", "origin", pushref], From b7dbd6db7871072e4d048d26d387539d5421191e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 3/7] Updating regex for URLs and SSH urls (fixing the later while at it) Add some more tests with it Signed-off-by: Pierre-Yves Chibon Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/utils.py b/pagure/utils.py index 142a994..d2c3852 100644 --- a/pagure/utils.py +++ b/pagure/utils.py @@ -319,13 +319,18 @@ urlregex = re.compile( # protocol identifier "(?:(?:https?|ftp|git)://)" # user:pass authentication - "(?:\S+(?::\S*)?@)?" "(?:" "(?P" + "(?:[-a-z\u00a1-\uffff0-9._~%!$&'()*+,;=:]+" + "(?::[-a-z0-9._~%!$&'()*+,;=:]*)?@)?" + "(?:" + "(?P" # IP address exclusion # private & local networks "(?:(?:10|127)" + ip_middle_octet + "{2}" + ip_last_octet + ")|" "(?:(?:169\.254|192\.168)" + ip_middle_octet + ip_last_octet + ")|" "(?:172\.(?:1[6-9]|2\d|3[0-1])" + ip_middle_octet + ip_last_octet + "))" "|" + # private & local hosts + "(?P" "(?:localhost))" "|" # IP address dotted notation octets # excludes loopback network 0.0.0.0 # excludes reserved space >= 224.0.0.0 @@ -336,6 +341,39 @@ urlregex = re.compile( "" + ip_middle_octet + "{2}" "" + ip_last_octet + ")" "|" + # IPv6 RegEx from https://stackoverflow.com/a/17871737 + "\[(" + # 1:2:3:4:5:6:7:8 + "([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|" + # 1:: 1:2:3:4:5:6:7:: + "([0-9a-fA-F]{1,4}:){1,7}:|" + # 1::8 1:2:3:4:5:6::8 1:2:3:4:5:6::8 + "([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|" + # 1::7:8 1:2:3:4:5::7:8 1:2:3:4:5::8 + "([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|" + # 1::6:7:8 1:2:3:4::6:7:8 1:2:3:4::8 + "([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|" + # 1::5:6:7:8 1:2:3::5:6:7:8 1:2:3::8 + "([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|" + # 1::4:5:6:7:8 1:2::4:5:6:7:8 1:2::8 + "([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|" + # 1::3:4:5:6:7:8 1::3:4:5:6:7:8 1::8 + "[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|" + # ::2:3:4:5:6:7:8 ::2:3:4:5:6:7:8 ::8 :: + ":((:[0-9a-fA-F]{1,4}){1,7}|:)|" + # fe80::7:8%eth0 fe80::7:8%1 + # (link-local IPv6 addresses with zone index) + "fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|" + "::(ffff(:0{1,4}){0,1}:){0,1}" + "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}" + # ::255.255.255.255 ::ffff:255.255.255.255 ::ffff:0:255.255.255.255 + # (IPv4-mapped IPv6 addresses and IPv4-translated addresses) + "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|" + "([0-9a-fA-F]{1,4}:){1,4}:" + "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}" + # 2001:db8:3:4::192.0.2.33 64:ff9b::192.0.2.33 + # (IPv4-Embedded IPv6 Address) + "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])" ")\]|" # host name "(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" # domain name @@ -345,7 +383,11 @@ urlregex = re.compile( # port number "(?::\d{2,5})?" # resource path - "(?:/\S*)?" "$", + "(?:/[-a-z\u00a1-\uffff0-9._~%!$&'()*+,;=:@/]*)?" + # query string + "(?:\?\S*)?" + # fragment + "(?:#\S*)?" "$", re.UNICODE | re.IGNORECASE, ) urlpattern = re.compile(urlregex) @@ -354,15 +396,20 @@ urlpattern = re.compile(urlregex) ssh_urlregex = re.compile( "^" # protocol identifier - "(?:(?:(git\+)?ssh)://)" - # user:pass authentication - "(?:\S+(?::\S*)?@)?" "(?:" "(?P" + "(?:(?:ssh|git\+ssh)://)?" + # user@ authentication + "[-a-z\u00a1-\uffff0-9._~%!$&'()*+,;=:]+@" + # Opening section about host + "(?:" # IP address exclusion + "(?P" # private & local networks "(?:(?:10|127)" + ip_middle_octet + "{2}" + ip_last_octet + ")|" "(?:(?:169\.254|192\.168)" + ip_middle_octet + ip_last_octet + ")|" "(?:172\.(?:1[6-9]|2\d|3[0-1])" + ip_middle_octet + ip_last_octet + "))" "|" + # private & local hosts + "(?P" "(?:localhost))" "|" # IP address dotted notation octets # excludes loopback network 0.0.0.0 # excludes reserved space >= 224.0.0.0 @@ -373,16 +420,55 @@ ssh_urlregex = re.compile( "" + ip_middle_octet + "{2}" "" + ip_last_octet + ")" "|" + # IPv6 RegEx from https://stackoverflow.com/a/17871737 + "\[(" + # 1:2:3:4:5:6:7:8 + "([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|" + # 1:: 1:2:3:4:5:6:7:: + "([0-9a-fA-F]{1,4}:){1,7}:|" + # 1::8 1:2:3:4:5:6::8 1:2:3:4:5:6::8 + "([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|" + # 1::7:8 1:2:3:4:5::7:8 1:2:3:4:5::8 + "([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|" + # 1::6:7:8 1:2:3:4::6:7:8 1:2:3:4::8 + "([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|" + # 1::5:6:7:8 1:2:3::5:6:7:8 1:2:3::8 + "([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|" + # 1::4:5:6:7:8 1:2::4:5:6:7:8 1:2::8 + "([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|" + # 1::3:4:5:6:7:8 1::3:4:5:6:7:8 1::8 + "[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|" + # ::2:3:4:5:6:7:8 ::2:3:4:5:6:7:8 ::8 :: + ":((:[0-9a-fA-F]{1,4}){1,7}|:)|" + # fe80::7:8%eth0 fe80::7:8%1 + # (link-local IPv6 addresses with zone index) + "fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|" + "::(ffff(:0{1,4}){0,1}:){0,1}" + "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}" + # ::255.255.255.255 ::ffff:255.255.255.255 ::ffff:0:255.255.255.255 + # (IPv4-mapped IPv6 addresses and IPv4-translated addresses) + "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|" + "([0-9a-fA-F]{1,4}:){1,4}:" + "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}" + # 2001:db8:3:4::192.0.2.33 64:ff9b::192.0.2.33 + # (IPv4-Embedded IPv6 Address) + "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])" ")\]|" # host name "(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" # domain name "(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*" # TLD identifier - "(?:\.(?:[a-z\u00a1-\uffff]{2,}))" ")" + "(?:\.(?:[a-z\u00a1-\uffff]{2,}))" + # Closing the entire section about host + ")" # port number "(?::\d{2,5})?" # resource path - "(?:/\S*)?" "$", + "(?:[:/][-a-z\u00a1-\uffff0-9._~%!$&'()*+,;=:@/]*)?" + # query string + "(?:\?\S*)?" + # fragment + "(?:#\S*)?" "$", re.UNICODE | re.IGNORECASE, ) ssh_urlpattern = re.compile(ssh_urlregex) diff --git a/tests/test_pagure_flask_ui_plugins_mirror.py b/tests/test_pagure_flask_ui_plugins_mirror.py index 137a803..bcc7538 100644 --- a/tests/test_pagure_flask_ui_plugins_mirror.py +++ b/tests/test_pagure_flask_ui_plugins_mirror.py @@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) import pagure.lib +import pagure.utils import tests @@ -35,6 +36,22 @@ class PagureFlaskPluginMirrortests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git(os.path.join(self.path, 'repos')) + def test_valid_ssh_url_pattern(self): + """ Check a number of valide ssh target that the pattern should let + through. + """ + entries = [ + 'ssh://user@host.lcl:/path/to/repo.git', + 'git@github.com:user/project.git', + 'ssh://user@host.org/target', + 'git+ssh://user@host.org/target', + 'git+ssh://user@host.lcl:/path/to/repo.git', + ] + for el in entries: + print(el) + self.assertIsNotNone(pagure.utils.ssh_urlpattern.match(el)) + + def test_plugin_mirror_no_csrf(self): """ Test setting up the mirror plugin with no csrf. """ From 55de1c30387cd882991318c13649f512023d89f9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 4/7] Fixes in the tests for the ssh URL pattern Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_flask_util.py b/tests/test_pagure_flask_util.py index 5eec180..da18950 100644 --- a/tests/test_pagure_flask_util.py +++ b/tests/test_pagure_flask_util.py @@ -34,14 +34,15 @@ class PagureUtilSSHPatterntests(tests.Modeltests): patterns = [ 'ssh://user@host.com/repo.git', 'git+ssh://user@host.com/repo.git', - 'ssh://host.com/repo.git' - 'git+ssh://host.com/repo.git', - 'ssh://127.0.0.1/repo.git', - 'git+ssh://127.0.0.1/repo.git', + 'ssh://user@host.lcl:/path/to/repo.git', + 'git@github.com:user/project.git', + 'ssh://user@host.org/target', + 'git+ssh://user@host.org/target', + 'git+ssh://user@host.lcl:/path/to/repo.git', ] for pattern in patterns: print(pattern) - self.assertTrue(ssh_urlpattern.match(pattern)) + self.assertIsNotNone(ssh_urlpattern.match(pattern)) def test_ssh_pattern_invalid(self): @@ -52,13 +53,17 @@ class PagureUtilSSHPatterntests(tests.Modeltests): 'https://user@host.com/repo.git', 'git+https://user@host.com/repo.git', 'ssh://localhost/repo.git', + 'ssh://host.com/repo.git', 'git+ssh://localhost/repo.git', 'ssh://0.0.0.0/repo.git', 'git+ssh://0.0.0.0/repo.git', + 'git+ssh://host.com/repo.git', + 'ssh://127.0.0.1/repo.git', + 'git+ssh://127.0.0.1/repo.git', ] for pattern in patterns: print(pattern) - self.assertFalse(ssh_urlpattern.match(pattern)) + self.assertIsNone(ssh_urlpattern.match(pattern)) if __name__ == '__main__': From 6d6656a8028dc620ac15fcd6920d49c973da2d0d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 5/7] Adjust unit-test for new output in the hooks Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_lib_git_auth.py b/tests/test_pagure_lib_git_auth.py index 3f67d8a..d50226a 100644 --- a/tests/test_pagure_lib_git_auth.py +++ b/tests/test_pagure_lib_git_auth.py @@ -108,7 +108,7 @@ class PagureLibGitAuthtests(tests.Modeltests): self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertIn( - "Remote hook declined the push: " + "Remote hook declined the push: Running hooks for hooktest\n" "Denied push for ref 'refs/heads/master' for user 'pingou'\n" "All changes have been rejected", output_text @@ -147,7 +147,7 @@ class PagureLibGitAuthtests(tests.Modeltests): self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertIn( - "Remote hook declined the push: " + "Remote hook declined the push: Running hooks for hooktest\n" "Denied push for ref 'refs/heads/master' for user 'pingou'\n" "All changes have been rejected", output_text From c267404226f984386a701d1a3eeab47d6827eddb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 6/7] Use gitolite's own mechanism to bypass the update hook Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 893fc69..70ebb60 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -992,7 +992,7 @@ class TemporaryClone(object): _log.debug("Opts: %s", opts) env = os.environ.copy() env["GL_USER"] = username - env["BYPASS_GITOLITE"] = "defined" + env["GL_BYPASS_ACCESS_CHECKS"] = "1" env.update(extra) out = subprocess.check_output( ["git"] + opts + ["push", "origin", pushref], From e1044342e24fd283215aa5730eb487a7a4baef6f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 26 2018 10:37:19 +0000 Subject: [PATCH 7/7] Ensure the plugin is active Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/plugins.py b/pagure/lib/plugins.py index b3a4dd3..aa31078 100644 --- a/pagure/lib/plugins.py +++ b/pagure/lib/plugins.py @@ -69,6 +69,6 @@ def get_enabled_plugins(project, with_default=False): for plugin in load("pagure.hooks", subclasses=BaseHook): if plugin.db_object and hasattr(project, plugin.backref): dbobj = getattr(project, plugin.backref) - if dbobj: + if dbobj and dbobj.active: enabled.append((plugin, dbobj)) return enabled