From 016f9258ca51e4ebc31273d5f5dfb187dbdf291e Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Feb 08 2018 11:06:34 +0000
Subject: [PATCH 1/2] Improve pagure's performance when facing very large diff or files


This commit introduce a new configuration key allowing to tune how large
diffs or files we support rendering in the UI.

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/pagure/default_config.py b/pagure/default_config.py
index 7c74374..662c818 100644
--- a/pagure/default_config.py
+++ b/pagure/default_config.py
@@ -88,6 +88,9 @@ ITEM_PER_PAGE = 48
 # Maximum size of the uploaded content
 MAX_CONTENT_LENGTH = 4 * 1024 * 1024  # 4 megabytes
 
+# Maximum size of diff/files above which we don't render in the UI
+MAX_FILE_UI = 1024 * 1024 * 4  # 4 megabytes
+
 # IP addresses allowed to access the internal endpoints
 IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1']
 
diff --git a/pagure/lib/mimetype.py b/pagure/lib/mimetype.py
index f70443b..f253f05 100644
--- a/pagure/lib/mimetype.py
+++ b/pagure/lib/mimetype.py
@@ -1,8 +1,12 @@
 # -*- coding: utf-8 -*-
 import logging
 import mimetypes
+import sys
+
 import kitchen.text.converters as ktc
+
 import pagure.lib.encoding_utils
+from pagure.config import config as pagure_config
 
 
 _log = logging.getLogger(__name__)
@@ -29,7 +33,9 @@ def guess_type(filename, data):
             else:
                 mimetype = 'text/plain'
 
-        if mimetype.startswith('text/') and not encoding:
+        size = sys.getsizeof(data)
+        if mimetype.startswith('text/') and not encoding \
+                and size < pagure_config['MAX_FILE_UI']:
             try:
                 encoding = pagure.lib.encoding_utils.guess_encoding(
                     ktc.to_bytes(data))
diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py
index 81c802e..a4776ed 100644
--- a/pagure/ui/filters.py
+++ b/pagure/ui/filters.py
@@ -16,6 +16,7 @@
 import datetime
 import textwrap
 import urlparse
+import sys
 
 import arrow
 import flask
@@ -409,6 +410,9 @@ def html_diff(diff, linenos='inline'):
     # the diff.
     difflexer.add_filter(VisibleWhitespaceFilter(wstokentype=False, tabs=True))
 
+    if sys.getsizeof(diff) > pagure_config['MAX_FILE_UI']:
+        return "This diff is too large and can't be rendered in the UI"
+
     return highlight(
         diff,
         difflexer,

From 9f4cc2c2fcc52fccd359dac99247fc5eaf176c46 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Feb 08 2018 11:06:34 +0000
Subject: [PATCH 2/2] Improve pagure's performance when render very large commit


The previous commit improved the situation for large diffs but for large
commits having a lot of small diffs, this did not work. In this commit
we are checking the size of the whole commit and then deciding whether
to display it or not.

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

---

diff --git a/pagure/templates/commit.html b/pagure/templates/commit.html
index c3c93ab..0a38429 100644
--- a/pagure/templates/commit.html
+++ b/pagure/templates/commit.html
@@ -204,9 +204,26 @@
         {% endautoescape %}
     {% endif %}
 </div>
-
 {% endfor %}
 
+{% if diff == [] %}
+<div class="card">
+  <div class="card-header">
+    <p>
+      This commit is too large to be rendered correctly in the UI, you
+      may be able to view it <a href="{{  url_for(
+        'ui_ns.view_raw_file',
+        repo=repo.name,
+        username=username,
+        namespace=repo.namespace,
+        identifier=commitid) }}">raw</a>,
+      or using git directly by running <pre>git show {{ commitid }}</pre>
+      in your clone of the repository.
+    </p>
+  </div>
+</div>
+{% endif %}
+
 {% endblock %}
 
 
diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py
index 8db2a44..9c9dd37 100644
--- a/pagure/ui/repo.py
+++ b/pagure/ui/repo.py
@@ -21,6 +21,7 @@ import datetime
 import json
 import logging
 import os
+import sys
 from cStringIO import StringIO
 from math import ceil
 
@@ -772,6 +773,13 @@ def view_commit(repo, commitid, username=None, namespace=None):
         # First commit in the repo
         diff = commit.tree.diff_to_tree(swap=True)
 
+    size = 0
+    for p in diff:
+        for h in p.hunks:
+            for l in h.lines:
+                size += sys.getsizeof(l.content)
+    if size  > (1024 * 1024 * 1024):
+        diff = []
     return flask.render_template(
         'commit.html',
         select='commits',