From 14eeca5e1186ab311a8e0fb5515ddf0f11635bbf Mon Sep 17 00:00:00 2001 From: Yanko Kaneti Date: Mar 02 2023 14:09:07 +0000 Subject: [PATCH 1/6] download-build: preserve build artefacts last modification time --- diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index 02c0cb9..565fb15 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -607,6 +607,11 @@ def download_file(url, relpath, quiet=False, noprogress=False, size=None, _download_progress(length, pos, filesize) if not length and not (quiet or noprogress): _download_progress(pos, pos, filesize) + last_modified = response.headers.get('last-modified') + if last_modified: + mtime = dateutil.parser.parse(last_modified) + if mtime: + os.utime(relpath, (time.time() ,time.mktime(mtime.astimezone().timetuple()))) finally: f.close() if pos == 0: From 037ac9d5753c627355859c4026d56b980218cb30 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 02 2023 14:16:41 +0000 Subject: [PATCH 2/6] fix py26 compatibility --- diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index 565fb15..7bd16a9 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -611,7 +611,11 @@ def download_file(url, relpath, quiet=False, noprogress=False, size=None, if last_modified: mtime = dateutil.parser.parse(last_modified) if mtime: - os.utime(relpath, (time.time() ,time.mktime(mtime.astimezone().timetuple()))) + # py 2.6 modifications, see koji.formatTimeLong + if mtime.tzinfo is None: + mtime = mtime.replace(tzinfo=dateutil.tz.gettz()) + mtime = mtime.astimezone(dateutil.tz.gettz()) + os.utime(relpath, (time.time(), time.mktime(mtime.timetuple()))) finally: f.close() if pos == 0: From 9b2bd92da22ed41bfc15cf68a4a84d0abd1ef4be Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 02 2023 14:17:12 +0000 Subject: [PATCH 3/6] fix tests Related: https://pagure.io/koji/issue/3711 --- diff --git a/tests/test_cli/test_download_file.py b/tests/test_cli/test_download_file.py index de3268f..05fe08f 100644 --- a/tests/test_cli/test_download_file.py +++ b/tests/test_cli/test_download_file.py @@ -56,9 +56,10 @@ class TestDownloadFile(unittest.TestCase): else: self.assertEqual(cm.exception.args, (21, 'Is a directory')) + @mock.patch('os.utime') @mock.patch('os.unlink') @mock_open() - def test_handle_download_file(self, m_open, os_unlink): + def test_handle_download_file(self, m_open, os_unlink, os_utime): self.reset_mock() m_open.return_value.tell.return_value = 0 rsp_head = self.head.return_value @@ -66,7 +67,10 @@ class TestDownloadFile(unittest.TestCase): rsp_head.headers = {'Content-Length': '5'} response = mock.MagicMock() self.get_mock.return_value = response - response.headers.get.return_value = '5' # content-length + response.headers.get.side_effect = [ + '5', # content-length + 'Thu, 02 Mar 2023 11:21:12 GMT', # last-modified + ] response.iter_content.return_value = ['abcde'] rv = download_file("http://url", self.filename) @@ -76,8 +80,7 @@ class TestDownloadFile(unittest.TestCase): self.assertMultiLineEqual(actual, expected) self.get_mock.assert_called_once() - m_open.assert_called_once() - response.headers.get.assert_not_called() + response.headers.get.assert_called_once() response.iter_content.assert_called_once() self.assertIsNone(rv) @@ -90,9 +93,9 @@ class TestDownloadFile(unittest.TestCase): rsp_head.status_code = 200 rsp_head.headers = {'Content-Length': str(65536 * 2)} response = mock.MagicMock() - self.get_mock.return_value = response response.headers.get.return_value = None # content-length response.iter_content.return_value = ['a' * 65536, 'b' * 65536] + self.get_mock.return_value = response rv = download_file("http://url", self.filename) @@ -103,19 +106,30 @@ class TestDownloadFile(unittest.TestCase): self.get_mock.assert_called_once() m_open.assert_called_once() - response.headers.get.assert_not_called() + response.headers.get.assert_called_once() response.iter_content.assert_called_once() self.assertIsNone(rv) def test_handle_download_file_with_size(self): + response = mock.MagicMock() + response.headers.get.side_effect = [ + '5', # content-length + '2022-01-01', # last-modified + ] + self.get_mock.return_value = response rv = download_file("http://url", self.filename, size=10, num=8) actual = self.stdout.getvalue() expected = 'Downloading [8/10]: %s\n\n' % self.filename self.assertMultiLineEqual(actual, expected) self.get_mock.assert_called_once() + self.assertEqual(len(response.headers.get.mock_calls), 2) self.assertIsNone(rv) def test_handle_download_file_quiet_noprogress(self): + response = mock.MagicMock() + response.headers.get.return_value = None + self.get_mock.return_value = response + download_file("http://url", self.filename, quiet=True, noprogress=False) actual = self.stdout.getvalue() expected = '' From ad87c08fa25708848413a01cddfe07dbea112d93 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 02 2023 14:40:22 +0000 Subject: [PATCH 4/6] RawHeader.get can return also string lists Related: https://pagure.io/koji/issue/3713 --- diff --git a/koji/__init__.py b/koji/__init__.py index 024a8c5..2490b7f 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -807,6 +807,14 @@ class RawHeader(object): elif dtype == 7: # raw data return self.header[pos:pos + count] + elif dtype == 8: + # string array + result = [] + for i in range(count): + end = self.header.find(six.b('\0'), pos) + result.append(self.header[pos:end]) + pos = end + 1 + return result else: # XXX - not all valid data types are handled raise GenericError("Unable to read header data type: %x" % dtype) From b6850ca89f0a49d7a1636df5b8c1137291b2f44b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 02 2023 14:40:22 +0000 Subject: [PATCH 5/6] also support get for unicode string arrays --- diff --git a/koji/__init__.py b/koji/__init__.py index 2490b7f..e88a677 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -770,7 +770,11 @@ class RawHeader(object): # unicode string array for i in range(count): end = self.header.find(six.b('\0'), pos) - print("i18n(%d): %r" % (end - pos, self.header[pos:end])) + try: + print("i18n(%d): %r" % (end - pos, _decode_item(self.header[pos:end]))) + except Exception: + print('INVALID STRING') + print("i18n(%d): %r" % (end - pos, self.header[pos:end])) pos = end + 1 next = pos else: @@ -815,6 +819,14 @@ class RawHeader(object): result.append(self.header[pos:end]) pos = end + 1 return result + elif dtype == 9: + # unicode string array + result = [] + for i in range(count): + end = self.header.find(six.b('\0'), pos) + result.append(_decode_item(self.header[pos:end])) + pos = end + 1 + return result else: # XXX - not all valid data types are handled raise GenericError("Unable to read header data type: %x" % dtype) From 686b7c5dc472ed1dfe25958886661a0906d360a3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 02 2023 14:40:22 +0000 Subject: [PATCH 6/6] attempt to decode strings in dump --- diff --git a/koji/__init__.py b/koji/__init__.py index e88a677..d0fece3 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -754,7 +754,12 @@ class RawHeader(object): elif dtype == 6: # string (null terminated) end = self.header.find(six.b('\0'), pos) - print("String(%d): %r" % (end - pos, self.header[pos:end])) + try: + print("String(%d): %r" % (end - pos, _decode_item(self.header[pos:end]))) + except ValueError: + print('INVALID STRING') + print("String(%d): %r" % (end - pos, self.header[pos:end])) + raise next = end + 1 elif dtype == 7: print("Data: %s" % hex_string(self.header[pos:pos + count]))