From 17903fcab5be990249cfd8c49968c995feab6a50 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 05 2019 10:36:27 +0000 Subject: [PATCH 1/4] refactor fixEncodingRecurse --- diff --git a/koji/__init__.py b/koji/__init__.py index 64ab90d..d18dd86 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -3122,47 +3122,49 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): return s +def fix_encoding2(value, fallback='iso8859-15', remove_nonprintable=False): + """ + Clean up a string + + This is similar to the original fixEncoding, but does not convert any + non-string values to '', and is written to work with util.DataWalker + + Mostly a no-op in python3, but remove_nonprintable is still honored + """ + + # play encoding tricks for py2 strings + if six.PY2: + if isinstance(value, six.text_type): + # value is already unicode, so just convert it + # to a utf8-encoded str + value = value.encode('utf8') + elif isinstance(value, six.binary_type): + # value is a str, but may be encoded in utf8 or some + # other non-ascii charset. Try to verify it's utf8, and if not, + # decode it using the fallback encoding. + try: + value = value.decode('utf8').encode('utf8') + except UnicodeDecodeError: + value = value.decode(fallback).encode('utf8') + + # remove nonprintable characters, if requested + if remove_nonprintable and isinstance(value, str): + # NOTE: we test for str instead of six.text_type deliberately + # - on py3, we're leaving bytes alone + # - on py2, we've just decoded any unicode + value = removeNonprintable(value) + + return value + + def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): """Recursively fix string encoding in an object - Similar behavior to fixEncoding, but recursive + This is simply fixEncoding2 recursively applied to an object """ - if six.PY3 and not remove_nonprintable: - # don't bother with fixing in py3 - return value - - if isinstance(value, tuple): - return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value]) - elif isinstance(value, list): - return [fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value] - elif isinstance(value, dict): - ret = {} - for k in value: - v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable) - k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable) - ret[k] = v - return ret - elif six.PY2 and isinstance(value, six.text_type): - if remove_nonprintable: - return removeNonprintable(value.encode('utf8')) - else: - return value.encode('utf8') - elif six.PY2 and isinstance(value, str): - # value is a str, but may be encoded in utf8 or some - # other non-ascii charset. Try to verify it's utf8, and if not, - # decode it using the fallback encoding. - try: - s = value.decode('utf8').encode('utf8') - except UnicodeDecodeError: - s = value.decode(fallback).encode('utf8') - if remove_nonprintable: - return removeNonprintable(s) - else: - return s - elif six.PY3 and isinstance(value, str) and remove_nonprintable: - return removeNonprintable(value) - else: - return value + kwargs = {'fallback': fallback, 'remove_nonprintable': remove_nonprintable} + walker = util.DataWalker(value, fix_encoding2, kwargs) + return walker.walk() def add_file_logger(logger, fn): From b9ad0a3551e9edce5c86653fc8ef7747e098860e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 05 2019 10:36:27 +0000 Subject: [PATCH 2/4] drop duplicate logic --- diff --git a/koji/__init__.py b/koji/__init__.py index d18dd86..aeb411c 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -3091,45 +3091,28 @@ def _fix_print(value): def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False): """ - Convert value to a 'str' object encoded as UTF-8. - If value is not valid UTF-8 to begin with, assume it is - encoded in the 'fallback' charset. - """ - if six.PY3: - if remove_nonprintable: - return removeNonprintable(value) - else: - return value + Compatibility wrapper for fix_encoding + Nontrue values are converted to the empty string, otherwise the result + is the same as fix_encoding. + """ if not value: - return six.b('') - - if isinstance(value, six.text_type): - # value is already unicode(py3: str), so just convert it - # to a utf8-encoded str(py3: bytes) - s = value.encode('utf8') - else: - # value is a str, but may be encoded in utf8 or some - # other non-ascii charset. Try to verify it's utf8, and if not, - # decode it using the fallback encoding. - try: - s = value.decode('utf8').encode('utf8') - except UnicodeDecodeError: - s = value.decode(fallback).encode('utf8') - if remove_nonprintable: - return removeNonprintable(s) - else: - return s + return '' + return fix_encoding(value, fallback, remove_nonprintable) -def fix_encoding2(value, fallback='iso8859-15', remove_nonprintable=False): +def fix_encoding(value, fallback='iso8859-15', remove_nonprintable=False): """ - Clean up a string + Adjust string to work around encoding issues + + In python2, unicode strings are encoded as utf8. For normal + strings, we attempt to fix encoding issues. The fallback option + is the encoding to use if the string is not valid utf8. - This is similar to the original fixEncoding, but does not convert any - non-string values to '', and is written to work with util.DataWalker + If remove_nonprintable is True, then nonprintable characters are + filtered out. - Mostly a no-op in python3, but remove_nonprintable is still honored + In python3 this is mostly a no-op, but remove_nonprintable is still honored """ # play encoding tricks for py2 strings @@ -3163,7 +3146,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): This is simply fixEncoding2 recursively applied to an object """ kwargs = {'fallback': fallback, 'remove_nonprintable': remove_nonprintable} - walker = util.DataWalker(value, fix_encoding2, kwargs) + walker = util.DataWalker(value, fix_encoding, kwargs) return walker.walk() From f641528667d9e1387f7df78c22e852a93029c2c6 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 05 2019 11:04:57 +0000 Subject: [PATCH 3/4] fix str-type testing in fix_encoding Fixes: https://pagure.io/koji/issue/1318 --- diff --git a/koji/__init__.py b/koji/__init__.py index aeb411c..b1ed768 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -3074,7 +3074,6 @@ def removeNonprintable(value): return value.translate(NONPRINTABLE_CHARS_TABLE) - def _fix_print(value): """Fix a string so it is suitable to print @@ -3117,11 +3116,10 @@ def fix_encoding(value, fallback='iso8859-15', remove_nonprintable=False): # play encoding tricks for py2 strings if six.PY2: - if isinstance(value, six.text_type): - # value is already unicode, so just convert it - # to a utf8-encoded str + if isinstance(value, unicode): + # just convert it to a utf8-encoded str value = value.encode('utf8') - elif isinstance(value, six.binary_type): + elif isinstance(value, str): # value is a str, but may be encoded in utf8 or some # other non-ascii charset. Try to verify it's utf8, and if not, # decode it using the fallback encoding. @@ -3143,7 +3141,7 @@ def fix_encoding(value, fallback='iso8859-15', remove_nonprintable=False): def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): """Recursively fix string encoding in an object - This is simply fixEncoding2 recursively applied to an object + This is simply fixEncoding recursively applied to an object """ kwargs = {'fallback': fallback, 'remove_nonprintable': remove_nonprintable} walker = util.DataWalker(value, fix_encoding, kwargs) diff --git a/tests/test_lib/test_fixEncoding.py b/tests/test_lib/test_fixEncoding.py index e53622e..ab26399 100644 --- a/tests/test_lib/test_fixEncoding.py +++ b/tests/test_lib/test_fixEncoding.py @@ -17,7 +17,7 @@ class FixEncodingTestCase(unittest.TestCase): """Main test case container""" simple_values = [ - # [ value, fixed ] + # [ unicode value, utf-8 encoded string ] ['', ''], [u'', ''], [u'góðan daginn', 'g\xc3\xb3\xc3\xb0an daginn'], @@ -51,6 +51,8 @@ class FixEncodingTestCase(unittest.TestCase): self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b) else: self.assertEqual(koji.fixEncoding(a), a) + d = a[:-3] + u'\x00\x01' + a[-3:] + self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), a) def test_fix_print(self): """Test the _fix_print function""" From 22669cdb30d6077354fc67271b7b302fadcf6318 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 07 2019 13:16:39 +0000 Subject: [PATCH 4/4] fix docstring --- diff --git a/koji/__init__.py b/koji/__init__.py index b1ed768..f4703eb 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -3141,7 +3141,7 @@ def fix_encoding(value, fallback='iso8859-15', remove_nonprintable=False): def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): """Recursively fix string encoding in an object - This is simply fixEncoding recursively applied to an object + This is simply fix_encoding recursively applied to an object """ kwargs = {'fallback': fallback, 'remove_nonprintable': remove_nonprintable} walker = util.DataWalker(value, fix_encoding, kwargs)