From 9dad14cfca3df026214e9eebe3c062336e202a46 Mon Sep 17 00:00:00 2001 From: Ryan Lerch Date: Jun 06 2016 08:27:03 +0000 Subject: [PATCH 1/3] Convert filenames with utf8 chars when outputting them in templates If a git repo has a filename in it with a utf8 encoded char, the template would previously fail to render with an error like: UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 1: ordinal not in range(128) This adds a filter to convert the utf8 charaters in the templates. The tests have also been updated to add a file to one of the repos with a non-ascii char. --- diff --git a/pagure/templates/commit.html b/pagure/templates/commit.html index 1a4643b..faa11e2 100644 --- a/pagure/templates/commit.html +++ b/pagure/templates/commit.html @@ -54,10 +54,10 @@ {% set filecount = filecount + 1 %} {% if patch | hasattr('new_file_path') %} - {{ patch.new_file_path }} + {{ patch.new_file_path | unicode}} {% elif patch | hasattr('delta') %} - {{ patch.delta.new_file.path }} + {{ patch.delta.new_file.path | unicode }} {% endif %}
{% if not patch.is_binary %} @@ -102,13 +102,13 @@ - {{ patch.new_file_path }} + {{ patch.new_file_path | unicode }} {% elif patch | hasattr('delta') %} - {{ patch.delta.new_file.path }} + {{ patch.delta.new_file.path | unicode }} {% endif %} {% if not patch.is_binary %} diff --git a/pagure/templates/pull_request.html b/pagure/templates/pull_request.html index ee90cb9..af3837b 100644 --- a/pagure/templates/pull_request.html +++ b/pagure/templates/pull_request.html @@ -349,7 +349,7 @@ identifier=identifier, filename=filepath) }}" {% endif %} - title="View file as of {{ patch_new_id|short }}">{{filepath}} + title="View file as of {{ patch_new_id|short }}">{{filepath | unicode}} {% endmacro %} {% if patch | hasattr('new_file_path') %} diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py index 28357bd..7462780 100644 --- a/pagure/ui/filters.py +++ b/pagure/ui/filters.py @@ -409,3 +409,7 @@ def text_wraps(dictionary): return max([int(k) for k in dictionary if k]) + 1 else: return 1 + +@APP.template_filter('unicode') +def convert_unicode(text): + return unicode(text, "utf8") diff --git a/tests/__init__.py b/tests/__init__.py index a5545a4..f39b1d9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -330,6 +330,9 @@ def add_content_git_repo(folder): with open(os.path.join(newfolder, subfolder, 'file'), 'w') as stream: stream.write('foo\n bar\nbaz') repo.index.add(os.path.join(subfolder, 'file')) + with open(os.path.join(newfolder, subfolder, 'fileŠ'), 'w') as stream: + stream.write('foo\n bar\nbaz') + repo.index.add(os.path.join(subfolder, 'fileŠ')) repo.index.write() # Commits the files added From f5f5d6cfbc4b98421673c31a74525a4fed8f5581 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 06 2016 09:05:59 +0000 Subject: [PATCH 2/3] Make the unicode filter be a little more py3 ready --- diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py index 7462780..5686c2e 100644 --- a/pagure/ui/filters.py +++ b/pagure/ui/filters.py @@ -410,6 +410,10 @@ def text_wraps(dictionary): else: return 1 + @APP.template_filter('unicode') def convert_unicode(text): - return unicode(text, "utf8") + if isinstance(text, basestring): + return text.decode("utf8") + else: + return text From 557220b8f14b3a7feb909118bc960f421f3502ef Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 06 2016 09:06:35 +0000 Subject: [PATCH 3/3] Check how is displayed the non-utf-8 file Adjust the templates so that the link are fine as well --- diff --git a/pagure/templates/file.html b/pagure/templates/file.html index f312966..788ac4e 100644 --- a/pagure/templates/file.html +++ b/pagure/templates/file.html @@ -32,14 +32,16 @@
{% for branch in branches %} {% if origin == 'view_tree' %} - + {{ branch }} {% elif origin == 'view_file' %} - + {{ branch }} {% endif %} @@ -52,11 +54,16 @@
-
    -
  1. +
  2. +   {{ branchname }}
    3. {% - for file in filename.split('/') %} + }}"> + +   {{ branchname }} + + + {% for file in filename.split('/') %} {% if loop.first %} {% set path = file %} {% else %} @@ -65,8 +72,15 @@ {% if loop.index != loop.length %}
  3.   {{ file }}
    4. {% elif file %}
  4.   {{ file }}
    5. {% endif %} + filename=path | unicode)}}"> +   {{ file }} + + {% elif file %} +
  5. + +   {{ file }} +
    6. + {% endif %} {% endfor %}
@@ -78,22 +92,27 @@ {% if output_type in ('file','binary','image','markup') %}
{% if output_type in ('file','markup') and repo_admin %} - Edit + Edit {% endif %} {% if output_type == 'markup' %} - Text + Text {% else %} - Blob + Blob {% endif %} - Raw + Raw
{% endif %} {% endif %} @@ -109,14 +128,14 @@ {% elif output_type == 'image' %} + filename=filename | unicode) }}"/> {% elif output_type == 'binary' %}

Binary files cannot be rendered.
Please view the raw version + filename=filename | unicode) }}">view the raw version

{% endif %} @@ -139,9 +158,9 @@ - {{ entry.name }} + filename=filename | unicode + '/' + + entry.name | unicode if filename else entry.name) }}"> + {{ entry.name | unicode }} {% if entry.filemode == 16384 %}{% endif%} diff --git a/tests/test_pagure_flask_ui_no_master_branch.py b/tests/test_pagure_flask_ui_no_master_branch.py index d843d86..b34f7b5 100644 --- a/tests/test_pagure_flask_ui_no_master_branch.py +++ b/tests/test_pagure_flask_ui_no_master_branch.py @@ -220,13 +220,19 @@ class PagureFlaskNoMasterBranchtests(tests.Modeltests): output = self.app.get('/test/blob/feature/f/sources') self.assertEqual(output.status_code, 200) self.assertIn( - '' - '  feature', - output.data) - self.assertIn( - '
  • ' - '  sources
    • ', - output.data) + ''' +
      +
    1. + + +   feature + +
      2. +
    2. + +   sources +
      3. +
    ''', output.data) self.assertTrue( # new version of pygments '
    foo
    ' in output.data diff --git a/tests/test_pagure_flask_ui_repo.py b/tests/test_pagure_flask_ui_repo.py index 8fd0b94..0adf303 100644 --- a/tests/test_pagure_flask_ui_repo.py +++ b/tests/test_pagure_flask_ui_repo.py @@ -1604,9 +1604,16 @@ index 0000000..fb7093d output = self.app.get('/test/tree/') self.assertEqual(output.status_code, 200) - self.assertTrue( - '
  • ' - '  None
    • ' in output.data) + self.assertIn( + ''' +
      +
    1. + + +   None + +
      2. +
    ''', output.data) self.assertTrue( 'No content found in this repository' in output.data) @@ -1666,6 +1673,13 @@ index 0000000..fb7093d self.assertFalse( 'No content found in this repository' in output.data) + output = self.app.get( + '/fork/pingou/test3/blob/master/f/folder1/folder2') + self.assertEqual(output.status_code, 200) + self.assertTrue( + '' in output.data) + @patch('pagure.lib.notify.send_email') @patch('pagure.ui.repo.admin_session_timedout') diff --git a/tests/test_pagure_flask_ui_slash_branch_name.py b/tests/test_pagure_flask_ui_slash_branch_name.py index 17ec396..c0abc07 100644 --- a/tests/test_pagure_flask_ui_slash_branch_name.py +++ b/tests/test_pagure_flask_ui_slash_branch_name.py @@ -209,11 +209,19 @@ class PagureFlaskSlashInBranchtests(tests.Modeltests): output = self.app.get('/test/blob/master/f/sources') self.assertEqual(output.status_code, 200) self.assertIn( - '
  •   master
    • ' - '
  • ' - '  sources
    • ', - output.data) + ''' +
      +
    1. + + +   master + +
      2. +
    2. + +   sources +
      3. +
    ''', output.data) output = self.app.get('/test/blob/master/f/.gitignore') self.assertEqual(output.status_code, 404) @@ -221,12 +229,19 @@ class PagureFlaskSlashInBranchtests(tests.Modeltests): output = self.app.get('/test/blob/maxamilion/feature/f/.gitignore') self.assertEqual(output.status_code, 200) self.assertIn( - '
  • ' - '' - '  maxamilion/feature
    • ' - '
  • ' - '  .gitignore
    • ', - output.data) + ''' +
      +
    1. + + +   maxamilion/feature + +
      2. +
    2. + +   .gitignore +
      3. +
    ''', output.data) self.assertTrue( # new version of pygments '
    *~
    ' in output.data