From b76be0fa21283d373d9f0edd86ecb970d00a4013 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Mar 13 2019 12:59:53 +0000 Subject: [PATCH 1/2] Stream the repoSpanner proxy responses This should make bigger replies work significantly better because requests won't need to keep the full thing in memory. Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/ui/clone.py b/pagure/ui/clone.py index 9fe5ffc..2c71b10 100644 --- a/pagure/ui/clone.py +++ b/pagure/ui/clone.py @@ -160,12 +160,25 @@ def proxy_repospanner(project, service): regionurl, regioninfo = project.repospanner_repo_info("main") url = "%s/%s" % (regionurl, oper) + # Older flask/werkzeug versions don't support both an input and output + # stream: this results in a blank upload. + # So, we optimize for the direction the majority of the data will likely + # flow. + streamargs = {} + if service == "git-receive-pack": + # This is a Push operation, optimize for data from the client + streamargs["data"] = flask.request.stream + streamargs["stream"] = False + else: + # This is a Pull operation, optimize for data from the server + streamargs["data"] = flask.request.data + streamargs["stream"] = True + resp = requests.request( flask.request.method, url, verify=regioninfo["ca"], cert=(regioninfo["push_cert"]["cert"], regioninfo["push_cert"]["key"]), - data=flask.request.stream, headers={ "Content-Encoding": flask.request.content_encoding, "Content-Type": flask.request.content_type, @@ -175,6 +188,7 @@ def proxy_repospanner(project, service): "x-Extra-project_user": project.user if project.is_fork else "", "X-Extra-project_namespace": project.namespace, }, + **streamargs ) # Strip out any headers that cause problems @@ -186,6 +200,7 @@ def proxy_repospanner(project, service): resp.iter_content(chunk_size=128), status=resp.status_code, headers=dict(resp.headers), + direct_passthrough=True, ) From 05f1d98b19a2e854d9f86c534c117de7a1f1501e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Mar 13 2019 12:59:53 +0000 Subject: [PATCH 2/2] Fix testing receive-pack in push Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/test_pagure_repospanner.py b/tests/test_pagure_repospanner.py index ae2346a..eadf6ef 100644 --- a/tests/test_pagure_repospanner.py +++ b/tests/test_pagure_repospanner.py @@ -356,8 +356,11 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): # run on a TCP port, which the test environment doesn't do. output = self.app.get('/clonetest.git/info/refs?service=git-upload-pack') self.assertEqual(output.status_code, 200) + self.assertEqual(output.content_type, + "application/x-git-upload-pack-advertisement") output_text = output.get_data(as_text=True) self.assertIn("# service=git-upload-pack", output_text) + self.assertIn("agent=repoSpanner", output_text) self.assertIn("symref=HEAD:refs/heads/master", output_text) self.assertIn(" refs/heads/master\x00", output_text) @@ -388,12 +391,15 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): # Unfortunately, actually testing a git clone would need the app to # run on a TCP port, which the test environment doesn't do. output = self.app.get( - '/clonetest.git/info/refs?service=git-upload-pack', + '/clonetest.git/info/refs?service=git-receive-pack', environ_overrides={'REMOTE_USER': 'pingou'}, ) self.assertEqual(output.status_code, 200) + self.assertEqual(output.content_type, + "application/x-git-receive-pack-advertisement") output_text = output.get_data(as_text=True) - self.assertIn("# service=git-upload-pack", output_text) + self.assertIn("# service=git-receive-pack", output_text) + self.assertIn("agent=repoSpanner", output_text) self.assertIn("symref=HEAD:refs/heads/master", output_text) self.assertIn(" refs/heads/master\x00", output_text)