From 6c0e3b29920af5fc736eba90da3bb34e497f2de6 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <patrick@puiterwijk.org>
Date: Oct 11 2018 18:34:30 +0000
Subject: Create pseudo repos by creating an initial clone


This makes sure that the initial set of objects get created as a packfile.
This massively improves repo handling with huge repos due to a bug in
libgit2 where it only closes object files when the repo is closed.

Signed-off-by: Patrick Uiterwijk <patrick@puiterwijk.org>

---

diff --git a/pagure/lib/git.py b/pagure/lib/git.py
index 3fa18aa..a9e82c7 100644
--- a/pagure/lib/git.py
+++ b/pagure/lib/git.py
@@ -900,42 +900,10 @@ class TemporaryClone(object):
             # pygit2.clone_repository does not have the "branches" attribute.
             self.repo = pygit2.Repository(self.repopath)
         else:
-            repourl, regioninfo = self._project.repospanner_repo_info(
-                self._repotype
+            self.repo = self._project._repospanner_clone(
+                self._repotype, False, self.repopath
             )
 
-            command = [
-                "git",
-                "-c",
-                "protocol.ext.allow=always",
-                "clone",
-                "ext::%s %s"
-                % (
-                    pagure_config["REPOBRIDGE_BINARY"],
-                    self._project._repospanner_repo_name(self._repotype),
-                ),
-                self.repopath,
-            ]
-            environ = os.environ.copy()
-            environ.update(
-                {
-                    "USER": "pagure",
-                    "REPOBRIDGE_CONFIG": ":environment:",
-                    "REPOBRIDGE_BASEURL": regioninfo["url"],
-                    "REPOBRIDGE_CA": regioninfo["ca"],
-                    "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"],
-                    "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"],
-                }
-            )
-            with open(os.devnull, "w") as devnull:
-                subprocess.check_call(
-                    command,
-                    stdout=devnull,
-                    stderr=subprocess.STDOUT,
-                    env=environ,
-                )
-            self.repo = pygit2.Repository(self.repopath)
-
         # Make sure that all remote refs are mapped to local ones.
         for branchname in self.repo.branches.remote:
             localname = branchname.replace("origin/", "")
diff --git a/pagure/lib/model.py b/pagure/lib/model.py
index 2725665..0c5524e 100644
--- a/pagure/lib/model.py
+++ b/pagure/lib/model.py
@@ -21,6 +21,7 @@ import json
 import operator
 import re
 import pygit2
+import subprocess
 import os
 
 import six
@@ -540,6 +541,49 @@ class Project(BASE):
             self.fullname,
         )
 
+    def _repospanner_clone(self, repotype, set_config, target):
+        """ Create a clone of a repoSpanner repo to filesystem.
+
+        """
+        repourl, regioninfo = self.repospanner_repo_info(repotype)
+
+        command = [
+            "git",
+            "-c",
+            "protocol.ext.allow=always",
+            "clone",
+            "ext::%s %s"
+            % (
+                pagure_config["REPOBRIDGE_BINARY"],
+                self._repospanner_repo_name(repotype),
+            ),
+            target,
+        ]
+        environ = os.environ.copy()
+        environ.update(
+            {
+                "USER": "pagure",
+                "REPOBRIDGE_CONFIG": ":environment:",
+                "REPOBRIDGE_BASEURL": regioninfo["url"],
+                "REPOBRIDGE_CA": regioninfo["ca"],
+                "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"],
+                "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"],
+            }
+        )
+        with open(os.devnull, "w") as devnull:
+            subprocess.check_call(
+                command, stdout=devnull, stderr=subprocess.STDOUT, env=environ
+            )
+
+        repo = pygit2.Repository(target)
+        if set_config:
+            repo.config["repospanner.url"] = repourl
+            repo.config["repospanner.cert"] = regioninfo["push_cert"]["cert"]
+            repo.config["repospanner.key"] = regioninfo["push_cert"]["key"]
+            repo.config["repospanner.cacert"] = regioninfo["ca"]
+            repo.config["repospanner.enabled"] = True
+        return repo
+
     def repopath(self, repotype):
         """ Return the full repository path of the git repo on the filesystem.
 
@@ -552,16 +596,7 @@ class Project(BASE):
                 pagure_config["REPOSPANNER_PSEUDO_FOLDER"], repotype, self.path
             )
             if not os.path.exists(pseudopath):
-                repourl, regioninfo = self.repospanner_repo_info(repotype)
-                fake = pygit2.init_repository(pseudopath, bare=True)
-                fake.config["repospanner.url"] = repourl
-                fake.config["repospanner.cert"] = regioninfo["push_cert"][
-                    "cert"
-                ]
-                fake.config["repospanner.key"] = regioninfo["push_cert"]["key"]
-                fake.config["repospanner.cacert"] = regioninfo["ca"]
-                fake.config["repospanner.enabled"] = True
-                del fake
+                self._repospanner_clone(repotype, True, pseudopath)
             return pseudopath
 
         maindir = None