From 45f88f450710f359ebd95650de75278e4d91c762 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jun 12 2024 21:49:06 +0000 Subject: [PATCH 1/3] fix tz mismatch issues with various queries --- diff --git a/kojihub/db.py b/kojihub/db.py index 40c121d..9d93925 100644 --- a/kojihub/db.py +++ b/kojihub/db.py @@ -23,6 +23,7 @@ from __future__ import absolute_import +import datetime import logging import koji import os @@ -293,6 +294,11 @@ def _singleRow(query, values, fields, strict=False): return None +def convert_timestamp(ts): + """Convert a numeric timestamp to a string suitable for a datetimetz field""" + return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc).isoformat(' ') + + def get_event(): """Get an event id for this transaction diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index a408491..70e4e12 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -94,6 +94,7 @@ from .db import ( # noqa: F401 get_event, nextval, currval, + convert_timestamp, ) @@ -6811,10 +6812,8 @@ class CG_Importer(object): buildinfo['build_id'] = metadata['build']['build_id'] # epoch is not in the metadata spec, but we allow it to be specified buildinfo['epoch'] = metadata['build'].get('epoch', None) - buildinfo['start_time'] = \ - datetime.datetime.fromtimestamp(float(metadata['build']['start_time'])).isoformat(' ') - buildinfo['completion_time'] = \ - datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ') + buildinfo['start_time'] = convert_timestamp(float(metadata['build']['start_time'])) + buildinfo['completion_time'] = convert_timestamp(float(metadata['build']['end_time'])) owner = metadata['build'].get('owner', None) # get task id from OSBS or from standard place buildinfo['task_id'] = self.get_task_id_from_metadata(metadata) @@ -8493,7 +8492,7 @@ def query_history(tables=None, **kwargs): fields['revoker.id = %(editor)i'] = '_revoked_by' elif arg == 'after': if not isinstance(value, str): - value = datetime.datetime.fromtimestamp(value).isoformat(' ') + value = convert_timestamp(value) data['after'] = value clauses.append('ev1.time > %(after)s OR ev2.time > %(after)s') fields['ev1.time > %(after)s'] = '_created_after' @@ -8509,7 +8508,7 @@ def query_history(tables=None, **kwargs): fields[r_test] = '_revoked_after_event' elif arg == 'before': if not isinstance(value, str): - value = datetime.datetime.fromtimestamp(value).isoformat(' ') + value = convert_timestamp(value) data['before'] = value clauses.append('ev1.time < %(before)s OR ev2.time < %(before)s') # clauses.append("date_part('epoch', ev1.time) < %(before)s OR " @@ -12289,19 +12288,19 @@ class RootExports(object): clauses.append('build.state = %(state)i') if createdBefore: if not isinstance(createdBefore, str): - createdBefore = datetime.datetime.fromtimestamp(createdBefore).isoformat(' ') + createdBefore = convert_timestamp(createdBefore) clauses.append('events.time < %(createdBefore)s') if createdAfter: if not isinstance(createdAfter, str): - createdAfter = datetime.datetime.fromtimestamp(createdAfter).isoformat(' ') + createdAfter = convert_timestamp(createdAfter) clauses.append('events.time > %(createdAfter)s') if completeBefore: if not isinstance(completeBefore, str): - completeBefore = datetime.datetime.fromtimestamp(completeBefore).isoformat(' ') + completeBefore = convert_timestamp(completeBefore) clauses.append('build.completion_time < %(completeBefore)s') if completeAfter: if not isinstance(completeAfter, str): - completeAfter = datetime.datetime.fromtimestamp(completeAfter).isoformat(' ') + completeAfter = convert_timestamp(completeAfter) clauses.append('build.completion_time > %(completeAfter)s') if cgID: cgID = lookup_name('content_generator', cgID, strict=False) @@ -13504,7 +13503,7 @@ class RootExports(object): if opts.get(key) is not None: value = opts[key] if not isinstance(value, str): - opts[key] = datetime.datetime.fromtimestamp(value).isoformat(' ') + opts[key] = convert_timestamp(value) conditions.append('%(field)s %(cmp)s %%(%(key)s)s' % locals()) query = QueryProcessor(columns=fields, aliases=aliases, tables=tables, joins=joins, From 50b8cff488245fc5504d5f9f48e8569541ecd084 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jun 12 2024 21:57:10 +0000 Subject: [PATCH 2/3] fix unit tests --- diff --git a/tests/test_hub/test_query_history.py b/tests/test_hub/test_query_history.py index 5d15602..e881a8e 100644 --- a/tests/test_hub/test_query_history.py +++ b/tests/test_hub/test_query_history.py @@ -259,7 +259,7 @@ class TestQueryHistory(DBQueryTestCase): "LEFT OUTER JOIN users AS revoker ON revoker.id = revoker_id", 'LEFT OUTER JOIN users ON user_id = users.id', 'LEFT OUTER JOIN content_generator ON cg_id = content_generator.id']) - self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40'}) + self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40+00:00'}) query = self.queries[1] self.assertEqual(query.tables, ['tag_package_owners']) @@ -284,7 +284,7 @@ class TestQueryHistory(DBQueryTestCase): 'LEFT OUTER JOIN package ON package_id = package.id', 'LEFT OUTER JOIN tag ON tag_id = tag.id', 'LEFT OUTER JOIN users AS owner ON owner = owner.id']) - self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40'}) + self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40+00:00'}) query = self.queries[2] self.assertEqual(query.tables, ['user_groups']) @@ -306,7 +306,7 @@ class TestQueryHistory(DBQueryTestCase): "LEFT OUTER JOIN users AS revoker ON revoker.id = revoker_id", 'LEFT OUTER JOIN users ON user_id = users.id', 'users AS usergroup ON group_id = usergroup.id']) - self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40'}) + self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40+00:00'}) query = self.queries[3] self.assertEqual(query.tables, ['user_perms']) @@ -328,7 +328,7 @@ class TestQueryHistory(DBQueryTestCase): "LEFT OUTER JOIN users AS revoker ON revoker.id = revoker_id", 'LEFT OUTER JOIN users ON user_id = users.id', 'LEFT OUTER JOIN permissions AS permission ON perm_id = permission.id']) - self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40'}) + self.assertEqual(query.values, {'affected_user_id': 159, 'before': '2023-04-19 17:23:40+00:00'}) def test_permission_and_after_key(self): self.get_perm_id.return_value = 66 @@ -356,7 +356,7 @@ class TestQueryHistory(DBQueryTestCase): "LEFT OUTER JOIN users AS revoker ON revoker.id = revoker_id", 'LEFT OUTER JOIN tag ON tag_id = tag.id', 'LEFT OUTER JOIN permissions AS permission ON perm_id = permission.id']) - self.assertEqual(query.values, {'after': '2023-04-19 17:23:40', 'perm_id': 66}) + self.assertEqual(query.values, {'after': '2023-04-19 17:23:40+00:00', 'perm_id': 66}) query = self.queries[1] self.assertEqual(query.tables, ['user_perms']) @@ -378,7 +378,7 @@ class TestQueryHistory(DBQueryTestCase): "LEFT OUTER JOIN users AS revoker ON revoker.id = revoker_id", 'LEFT OUTER JOIN users ON user_id = users.id', 'LEFT OUTER JOIN permissions AS permission ON perm_id = permission.id']) - self.assertEqual(query.values, {'after': '2023-04-19 17:23:40', 'perm_id': 66}) + self.assertEqual(query.values, {'after': '2023-04-19 17:23:40+00:00', 'perm_id': 66}) def test_cg_key(self): self.lookup_name.return_value = {'id': 147} From 89d70a68e9fdb9696936d3fe4155f06fcd6bbf95 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jun 13 2024 14:44:24 +0000 Subject: [PATCH 3/3] use tzutc from dateutil --- diff --git a/kojihub/db.py b/kojihub/db.py index 9d93925..ebf36c6 100644 --- a/kojihub/db.py +++ b/kojihub/db.py @@ -40,6 +40,7 @@ import time import traceback import psycopg2 +from dateutil.tz import tzutc import koji.context context = koji.context.context @@ -296,7 +297,7 @@ def _singleRow(query, values, fields, strict=False): def convert_timestamp(ts): """Convert a numeric timestamp to a string suitable for a datetimetz field""" - return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc).isoformat(' ') + return datetime.datetime.fromtimestamp(ts, tzutc()).isoformat(' ') def get_event():