From 1c7e9a1a99eebe11ef2530525c27e5b4d24dbcbb Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 25 2023 14:42:54 +0000 Subject: [PATCH 1/4] kojira: prioritize awaited repos Related: https://pagure.io/koji/issue/3757 --- diff --git a/util/kojira b/util/kojira index 7c0d16d..f27691c 100755 --- a/util/kojira +++ b/util/kojira @@ -795,14 +795,24 @@ class RepoManager(object): if age < 0: self.logger.warning("Needed tag has future expire_ts: %r", entry) age = 0 - entry['score'] = age * adj + entry['score'] = age * adj * entry['score_adjust'] self.logger.debug("Needed tag %s got score %.2f", entry['taginfo']['name'], entry['score']) # so a day old unused repo gets about the regen same score as a # 2.4-hour-old, very popular repo def updateTagScores(self): - for entry in list(self.needed_tags.values()): + # call listTasks waitrepo + awaited = self.session.listTasks(opts={'method': ['waitrepo'], + 'state': [koji.TASK_STATES['FREE'], + koji.TASK_STATES['ASSIGNED'], + koji.TASK_STATES['OPEN']]}) + awaited = {koji.parse_task_params('waitrepo', task['request'])['tag'] for task in awaited} + for tag_id, entry in self.needed_tags.items(): + if tag_id in awaited: + # score multiplication factor, prioritize tags which are being awaited for + # not needed on every setTagScore call (initial point will not account it for) + entry['score_adjust'] = 1.5 self.setTagScore(entry) def _delete_needed_tag(self, tag_id): @@ -938,6 +948,7 @@ class RepoManager(object): 'taginfo': taginfo, 'expire_ts': ts, 'needed_since': time.time(), + 'score_adjust': 1.0, # modifier, updated in updateTagScores } self.setTagScore(entry) self.needed_tags[tag_id] = entry From b1acd55d84bddef41dda4294b602d79cf80f5be7 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Apr 25 2023 14:42:54 +0000 Subject: [PATCH 2/4] error handling --- diff --git a/util/kojira b/util/kojira index f27691c..ce06d6c 100755 --- a/util/kojira +++ b/util/kojira @@ -795,7 +795,7 @@ class RepoManager(object): if age < 0: self.logger.warning("Needed tag has future expire_ts: %r", entry) age = 0 - entry['score'] = age * adj * entry['score_adjust'] + entry['score'] = age * adj * entry.get('score_adjust', 1) self.logger.debug("Needed tag %s got score %.2f", entry['taginfo']['name'], entry['score']) # so a day old unused repo gets about the regen same score as a @@ -807,12 +807,20 @@ class RepoManager(object): 'state': [koji.TASK_STATES['FREE'], koji.TASK_STATES['ASSIGNED'], koji.TASK_STATES['OPEN']]}) - awaited = {koji.parse_task_params('waitrepo', task['request'])['tag'] for task in awaited} + awaited = set() + for task in awaited: + try: + awaited.add(koji.parse_task_params('waitrepo', task['request'])['tag']) + except Exception: + # ignore malformed tasks + self.logger.debug(f"Malformed task: {task}") + pass + for tag_id, entry in self.needed_tags.items(): if tag_id in awaited: # score multiplication factor, prioritize tags which are being awaited for # not needed on every setTagScore call (initial point will not account it for) - entry['score_adjust'] = 1.5 + entry['score_adjust'] = 2 self.setTagScore(entry) def _delete_needed_tag(self, tag_id): @@ -948,7 +956,6 @@ class RepoManager(object): 'taginfo': taginfo, 'expire_ts': ts, 'needed_since': time.time(), - 'score_adjust': 1.0, # modifier, updated in updateTagScores } self.setTagScore(entry) self.needed_tags[tag_id] = entry From f58e6ad92bbfdc81db56f16dab72cbbcb1622da8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Apr 25 2023 14:42:54 +0000 Subject: [PATCH 3/4] bugfixes and additional debug logging --- diff --git a/util/kojira b/util/kojira index ce06d6c..4c5f7dc 100755 --- a/util/kojira +++ b/util/kojira @@ -803,18 +803,26 @@ class RepoManager(object): def updateTagScores(self): # call listTasks waitrepo - awaited = self.session.listTasks(opts={'method': ['waitrepo'], - 'state': [koji.TASK_STATES['FREE'], - koji.TASK_STATES['ASSIGNED'], - koji.TASK_STATES['OPEN']]}) + tasks = self.session.listTasks(opts={'method': ['waitrepo'], + 'decode': True, + 'state': [koji.TASK_STATES['FREE'], + koji.TASK_STATES['ASSIGNED'], + koji.TASK_STATES['OPEN']]}) awaited = set() - for task in awaited: + if tasks: + logger.debug("Checking %s active waitrepo tasks", len(tasks)) + for task in tasks: try: - awaited.add(koji.parse_task_params('waitrepo', task['request'])['tag']) + tag_param = koji.parse_task_params('waitrepo', task['request'])['tag'] + taginfo = getTag(self.session, tag_param) except Exception: # ignore malformed tasks self.logger.debug(f"Malformed task: {task}") - pass + continue + if taginfo: + awaited.add(taginfo['id']) + else: + self.logger.debug("Found waitrepo task %i with bad tag arg: %r", task['id'], tag_param) for tag_id, entry in self.needed_tags.items(): if tag_id in awaited: From f112e96f97dcd7f6a7654a6faffde1d5ee326c2b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Apr 25 2023 14:49:20 +0000 Subject: [PATCH 4/4] flake8 --- diff --git a/util/kojira b/util/kojira index 4c5f7dc..aeb4078 100755 --- a/util/kojira +++ b/util/kojira @@ -822,7 +822,8 @@ class RepoManager(object): if taginfo: awaited.add(taginfo['id']) else: - self.logger.debug("Found waitrepo task %i with bad tag arg: %r", task['id'], tag_param) + self.logger.debug("Found waitrepo task %i with bad tag arg: %r", + task['id'], tag_param) for tag_id, entry in self.needed_tags.items(): if tag_id in awaited: