From 43b173bb17b5fb2044cf12cf4af25383253fa642 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 20 2024 12:48:58 +0000 Subject: [PATCH 1/7] --limit from scheduler-logs/info Related: https://pagure.io/koji/issue/4042 --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index a7db87d..c243b51 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -7992,6 +7992,8 @@ def anon_handle_scheduler_info(goptions, session, args): parser.add_option("--state", action="store", type='choice', default=None, choices=[x for x in koji.TASK_STATES.keys()], help="Limit data to task state") + parser.add_option("--limit", action="store", type=int, default=100, + help="Limit data to last N items [default: %default]") (options, args) = parser.parse_args(args) if len(args) > 0: parser.error("This command takes no arguments") @@ -8013,10 +8015,13 @@ def anon_handle_scheduler_info(goptions, session, args): clauses.append(('host_id', options.host)) if options.state: clauses.append(('state', koji.TASK_STATES[options.state])) + # reverse order, so we can apply limit if needed + opts = {'order': '-id', 'limit': options.limit} runs = session.scheduler.getTaskRuns( clauses=clauses, - fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts') + fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts'), + opts=opts ) mask = '%(task_id)-9s %(host_name)-20s %(state)-7s ' \ '%(create_ts)-17s %(start_ts)-17s %(completion_ts)-17s' @@ -8031,7 +8036,7 @@ def anon_handle_scheduler_info(goptions, session, args): } print(header) print('-' * len(header)) - for run in runs: + for run in reversed(runs): run['state'] = koji.TASK_STATES[run['state']] for ts in ('create_ts', 'start_ts', 'completion_ts'): run[ts] = _format_ts(run[ts]) @@ -8058,9 +8063,13 @@ def handle_scheduler_logs(goptions, session, args): help="Logs from given timestamp") parser.add_option("--to", type="float", action="store", dest="to_ts", help="Logs until given timestamp (included)") + parser.add_option("--limit", action="store", type=int, default=None, + help="Limit data to last N items. [default: %default]") (options, args) = parser.parse_args(args) if len(args) != 0: parser.error("There are no arguments for this command") + if options.from_ts and options.to_ts and options.limit: + parser.error("If both --from and --to are used, --limit shouldn't be used") clauses = [] if options.task: @@ -8076,12 +8085,18 @@ def handle_scheduler_logs(goptions, session, args): if options.to_ts: clauses.append(['msg_ts', '<', options.to_ts]) - logs = session.scheduler.getLogMessages(clauses, fields=('task_id', 'host_id', 'host_name', - 'msg_ts', 'msg')) + # reverse order, so we can apply limit if needed + opts = {'order': '-id'} + if options.limit is not None: + opts['limit'] = options.limit + logs = session.scheduler.getLogMessages(clauses, + fields=('task_id', 'host_id', 'host_name', + 'msg_ts', 'msg'), + opts=opts) for log in logs: log['time'] = time.asctime(time.localtime(log['msg_ts'])) - mask = ("%(task_id)s\t%(host_name)s\t%(time)s\t%(msg)s") + mask = ("%(task_id)-10s %(host_name)-20s %(time)-25s %(msg)-30s") if not goptions.quiet: h = mask % { 'task_id': 'Task', @@ -8092,7 +8107,7 @@ def handle_scheduler_logs(goptions, session, args): print(h) print('-' * len(h)) - for log in logs: + for log in reversed(logs): print(mask % log) diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py index d80f8fd..8aa7386 100644 --- a/kojihub/scheduler.py +++ b/kojihub/scheduler.py @@ -52,8 +52,10 @@ class LogMessagesQuery(QueryView): default_fields = ('id', 'task_id', 'host_id', 'msg', 'msg_ts') -def get_log_messages(clauses=None, fields=None): - return LogMessagesQuery(clauses, fields, opts={'order': 'id'}).execute() +def get_log_messages(clauses=None, fields=None, opts=None): + if opts is None: + opts = {'order': 'id'} + return LogMessagesQuery(clauses, fields, opts).execute() def get_tasks_for_host(hostID, retry=True): @@ -190,8 +192,10 @@ class TaskRunsQuery(QueryView): default_fields = ('id', 'task_id', 'host_id', 'active', 'create_ts') -def get_task_runs(clauses=None, fields=None): - return TaskRunsQuery(clauses, fields).execute() +def get_task_runs(clauses=None, fields=None, opts=None): + if opts is None: + opts = {'order': 'id'} + return TaskRunsQuery(clauses, fields, opts).execute() class TaskScheduler(object): From f3306a371b4e839a48459d510b84741f4dd7e0c4 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 20 2024 12:48:58 +0000 Subject: [PATCH 2/7] backward compatibility --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index c243b51..34511d1 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8015,14 +8015,28 @@ def anon_handle_scheduler_info(goptions, session, args): clauses.append(('host_id', options.host)) if options.state: clauses.append(('state', koji.TASK_STATES[options.state])) - # reverse order, so we can apply limit if needed - opts = {'order': '-id', 'limit': options.limit} - runs = session.scheduler.getTaskRuns( - clauses=clauses, - fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts'), - opts=opts - ) + + runs = None + if options.limit is not None: + try: + runs = session.scheduler.getTaskRuns( + clauses=clauses, + fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts'), + opts={'order': '-id', 'limit': options.limit} + ) + except koji.GenericError: + # iterator is sufficient here as we don't modify the list + runs = reversed(runs) + if runs is None: + # hub could be 1.34.0 without opts support or user doesn't use --limit + runs = session.scheduler.getTaskRuns( + clauses=clauses, + fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts') + ) + if options.limit: + runs = runs[-options.limit:] + mask = '%(task_id)-9s %(host_name)-20s %(state)-7s ' \ '%(create_ts)-17s %(start_ts)-17s %(completion_ts)-17s' if not goptions.quiet: @@ -8036,7 +8050,7 @@ def anon_handle_scheduler_info(goptions, session, args): } print(header) print('-' * len(header)) - for run in reversed(runs): + for run in runs: run['state'] = koji.TASK_STATES[run['state']] for ts in ('create_ts', 'start_ts', 'completion_ts'): run[ts] = _format_ts(run[ts]) @@ -8086,13 +8100,25 @@ def handle_scheduler_logs(goptions, session, args): clauses.append(['msg_ts', '<', options.to_ts]) # reverse order, so we can apply limit if needed - opts = {'order': '-id'} + logs = None if options.limit is not None: - opts['limit'] = options.limit - logs = session.scheduler.getLogMessages(clauses, - fields=('task_id', 'host_id', 'host_name', - 'msg_ts', 'msg'), - opts=opts) + try: + logs = session.scheduler.getLogMessages(clauses, + fields=('task_id', 'host_id', 'host_name', + 'msg_ts', 'msg'), + opts={'order': '-id', 'limit': options.limit}) + # don't use reversed() as it will be exhausted after modification loop later + logs.reverse() + except koji.GenericError: + pass + if logs is None: + # hub could be 1.34.0 without opts support or user doesn't use --limit + logs = session.scheduler.getLogMessages(clauses, + fields=('task_id', 'host_id', 'host_name', + 'msg_ts', 'msg')) + if options.limit: + logs = logs[-options.limit:] + for log in logs: log['time'] = time.asctime(time.localtime(log['msg_ts'])) @@ -8107,7 +8133,7 @@ def handle_scheduler_logs(goptions, session, args): print(h) print('-' * len(h)) - for log in reversed(logs): + for log in logs: print(mask % log) From fe3474c0e8e9a0d665f8c77a3dae4cd8c900daf2 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 20 2024 12:48:58 +0000 Subject: [PATCH 3/7] better compatibility errors --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 34511d1..cffb8ce 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8025,9 +8025,15 @@ def anon_handle_scheduler_info(goptions, session, args): fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts'), opts={'order': '-id', 'limit': options.limit} ) - except koji.GenericError: # iterator is sufficient here as we don't modify the list runs = reversed(runs) + except koji.ParameterError: + # 1.34.0 hub, we will try again with no opts usage + pass + except koji.GenericError as ex: + if 'Invalid method' in str(ex): + error("Hub version is %s and doesn't support scheduler methods " + "introduced in 1.34." % session.hub_version_str) if runs is None: # hub could be 1.34.0 without opts support or user doesn't use --limit runs = session.scheduler.getTaskRuns( @@ -8109,8 +8115,13 @@ def handle_scheduler_logs(goptions, session, args): opts={'order': '-id', 'limit': options.limit}) # don't use reversed() as it will be exhausted after modification loop later logs.reverse() - except koji.GenericError: + except koji.ParameterError: + # 1.34.0 hub, we will try again with no opts usage pass + except koji.GenericError as ex: + if 'Invalid method' in str(ex): + error("Hub version is %s and doesn't support scheduler methods " + "introduced in 1.34." % session.hub_version_str) if logs is None: # hub could be 1.34.0 without opts support or user doesn't use --limit logs = session.scheduler.getLogMessages(clauses, From 030a54b8d4f80c6fb14c20c8a52c3b42cc29d1cf Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 20 2024 12:48:58 +0000 Subject: [PATCH 4/7] use hub version check for compat code --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index cffb8ce..2c53f83 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8016,32 +8016,26 @@ def anon_handle_scheduler_info(goptions, session, args): if options.state: clauses.append(('state', koji.TASK_STATES[options.state])) + fields = ('id', 'task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts') + kwargs = {'clauses': clauses, 'fields': fields} + if session.hub_version < (1, 34, 0): + error("Hub version is %s and doesn't support scheduler methods " + "introduced in 1.34." % session.hub_version_str) + if options.limit is not None: + if session.hub_version > (1, 34, 1): + kwargs['opts'] = {'order': '-id', 'limit': options.limit} + runs = session.scheduler.getTaskRuns(**kwargs) - runs = None if options.limit is not None: - try: - runs = session.scheduler.getTaskRuns( - clauses=clauses, - fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts'), - opts={'order': '-id', 'limit': options.limit} - ) - # iterator is sufficient here as we don't modify the list + if session.hub_version > (1, 34, 1): + # server did it for us, but we need to reverse runs = reversed(runs) - except koji.ParameterError: - # 1.34.0 hub, we will try again with no opts usage - pass - except koji.GenericError as ex: - if 'Invalid method' in str(ex): - error("Hub version is %s and doesn't support scheduler methods " - "introduced in 1.34." % session.hub_version_str) - if runs is None: - # hub could be 1.34.0 without opts support or user doesn't use --limit - runs = session.scheduler.getTaskRuns( - clauses=clauses, - fields=('task_id', 'host_name', 'state', 'create_ts', 'start_ts', 'completion_ts') - ) - if options.limit: + else: + # emulate limit runs = runs[-options.limit:] + if session.hub_version < (1, 34, 1): + # emulate order + runs.sort(key=lambda r:r['id']) mask = '%(task_id)-9s %(host_name)-20s %(state)-7s ' \ '%(create_ts)-17s %(start_ts)-17s %(completion_ts)-17s' @@ -8088,7 +8082,7 @@ def handle_scheduler_logs(goptions, session, args): (options, args) = parser.parse_args(args) if len(args) != 0: parser.error("There are no arguments for this command") - if options.from_ts and options.to_ts and options.limit: + if options.from_ts and options.to_ts and options.limit is not None: parser.error("If both --from and --to are used, --limit shouldn't be used") clauses = [] @@ -8105,30 +8099,27 @@ def handle_scheduler_logs(goptions, session, args): if options.to_ts: clauses.append(['msg_ts', '<', options.to_ts]) - # reverse order, so we can apply limit if needed - logs = None + fields=('id', 'task_id', 'host_id', 'host_name', 'msg_ts', 'msg') + kwargs = {'clauses': clauses, 'fields': fields} + if session.hub_version < (1, 34, 0): + error("Hub version is %s and doesn't support scheduler methods " + "introduced in 1.34." % session.hub_version_str) if options.limit is not None: - try: - logs = session.scheduler.getLogMessages(clauses, - fields=('task_id', 'host_id', 'host_name', - 'msg_ts', 'msg'), - opts={'order': '-id', 'limit': options.limit}) + if session.hub_version > (1, 34, 1): + kwargs['opts'] = {'order': '-id', 'limit': options.limit} + logs = session.scheduler.getLogMessages(**kwargs) + + if options.limit is not None: + if session.hub_version > (1, 34, 1): + # server did it for us, but we need to reverse # don't use reversed() as it will be exhausted after modification loop later - logs.reverse() - except koji.ParameterError: - # 1.34.0 hub, we will try again with no opts usage - pass - except koji.GenericError as ex: - if 'Invalid method' in str(ex): - error("Hub version is %s and doesn't support scheduler methods " - "introduced in 1.34." % session.hub_version_str) - if logs is None: - # hub could be 1.34.0 without opts support or user doesn't use --limit - logs = session.scheduler.getLogMessages(clauses, - fields=('task_id', 'host_id', 'host_name', - 'msg_ts', 'msg')) - if options.limit: + logs = logs.reverse() + else: + # emulate limit logs = logs[-options.limit:] + if session.hub_version < (1, 34, 1): + # emulate order + logs.sort(key=lambda r:r['id']) for log in logs: log['time'] = time.asctime(time.localtime(log['msg_ts'])) From 2468cb989d22451b3022e5f37a67ec9ecbcbfdf5 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 20 2024 12:48:59 +0000 Subject: [PATCH 5/7] modify for 1.34.1 --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 2c53f83..7bd7932 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8022,12 +8022,12 @@ def anon_handle_scheduler_info(goptions, session, args): error("Hub version is %s and doesn't support scheduler methods " "introduced in 1.34." % session.hub_version_str) if options.limit is not None: - if session.hub_version > (1, 34, 1): + if session.hub_version >= (1, 34, 1): kwargs['opts'] = {'order': '-id', 'limit': options.limit} runs = session.scheduler.getTaskRuns(**kwargs) if options.limit is not None: - if session.hub_version > (1, 34, 1): + if session.hub_version >= (1, 34, 1): # server did it for us, but we need to reverse runs = reversed(runs) else: @@ -8105,12 +8105,12 @@ def handle_scheduler_logs(goptions, session, args): error("Hub version is %s and doesn't support scheduler methods " "introduced in 1.34." % session.hub_version_str) if options.limit is not None: - if session.hub_version > (1, 34, 1): + if session.hub_version >= (1, 34, 1): kwargs['opts'] = {'order': '-id', 'limit': options.limit} logs = session.scheduler.getLogMessages(**kwargs) if options.limit is not None: - if session.hub_version > (1, 34, 1): + if session.hub_version >= (1, 34, 1): # server did it for us, but we need to reverse # don't use reversed() as it will be exhausted after modification loop later logs = logs.reverse() From 2c2e9209d16fa5df6f7e61338f28cd9896a39a84 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 20 2024 13:11:43 +0000 Subject: [PATCH 6/7] fix typo --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 7bd7932..6f2550e 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8113,7 +8113,7 @@ def handle_scheduler_logs(goptions, session, args): if session.hub_version >= (1, 34, 1): # server did it for us, but we need to reverse # don't use reversed() as it will be exhausted after modification loop later - logs = logs.reverse() + logs.reverse() else: # emulate limit logs = logs[-options.limit:] From 2c9f69524acef4ca159cfaf0832b063a276f3a38 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 26 2024 11:27:31 +0000 Subject: [PATCH 7/7] fix flake8 --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 6f2550e..95189a2 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8035,7 +8035,7 @@ def anon_handle_scheduler_info(goptions, session, args): runs = runs[-options.limit:] if session.hub_version < (1, 34, 1): # emulate order - runs.sort(key=lambda r:r['id']) + runs.sort(key=lambda r: r['id']) mask = '%(task_id)-9s %(host_name)-20s %(state)-7s ' \ '%(create_ts)-17s %(start_ts)-17s %(completion_ts)-17s' @@ -8099,7 +8099,7 @@ def handle_scheduler_logs(goptions, session, args): if options.to_ts: clauses.append(['msg_ts', '<', options.to_ts]) - fields=('id', 'task_id', 'host_id', 'host_name', 'msg_ts', 'msg') + fields = ('id', 'task_id', 'host_id', 'host_name', 'msg_ts', 'msg') kwargs = {'clauses': clauses, 'fields': fields} if session.hub_version < (1, 34, 0): error("Hub version is %s and doesn't support scheduler methods " @@ -8119,7 +8119,7 @@ def handle_scheduler_logs(goptions, session, args): logs = logs[-options.limit:] if session.hub_version < (1, 34, 1): # emulate order - logs.sort(key=lambda r:r['id']) + logs.sort(key=lambda r: r['id']) for log in logs: log['time'] = time.asctime(time.localtime(log['msg_ts']))