From bb9d86db8b30ceb035fdbcf73a9103e8a62842bb Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 13 2023 13:19:14 +0000 Subject: [PATCH 1/4] create initial repo for sidetag Related: https://pagure.io/koji/issue/3808 --- diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 85c12af..9b5e824 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -2,6 +2,10 @@ # # SPDX-License-Identifier: GPL-2.0-or-later +import json +import logging +import os +import shutil import koji from koji.context import context from koji.plugin import callback, export @@ -20,9 +24,16 @@ from kojihub import ( get_user, policy_get_user, readInheritanceData, + repo_init, + repo_problem, + repo_ready, + RootExports, ) from kojihub.db import QueryProcessor, nextval +rootexports = RootExports() + +logger = logging.getLogger('koji.plugin.sidetag') CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" CONFIG = None @@ -87,6 +98,7 @@ def createSideTag(basetag, debuginfo=False, suffix=None): :returns dict: sidetag name + id """ + logger.debug(f'Creating sidetag for {basetag}') if suffix and suffix not in ALLOWED_SUFFIXES: raise koji.GenericError("%s suffix is not allowed for sidetag" % suffix) @@ -136,7 +148,70 @@ def createSideTag(basetag, debuginfo=False, suffix=None): ) _create_build_target(sidetag_name, sidetag_id, sidetag_id) - return {"name": sidetag_name, "id": sidetag_id} + # Sidetag is ready now, we try to create initial repo but if it is not successful + # kojira will take proper care later + return_data = {"name": sidetag_name, "id": sidetag_id} + + # In case it it is not maven repo, we can copy old one and don't wait for kojira + # for maven and/or src, leave it on kojira + if basetag.get('maven_support'): + # maven repos contains too many files to be handled sync in hub call, let kojira do that + logger.debug(f'Not creating initial repo for {sidetag_name} due to maven_support') + return return_data + + # find active basetag repo + repo_info = rootexports.getRepo(basetag, state=koji.REPO_READY) + if not repo_info: + return return_data + + src_repo = koji.pathinfo.repo(repo_info['id'], basetag['name']) + try: + repo_json = json.load(open(os.path.join(src_repo, 'repo.json'))) + except IOError: + logger.debug("Can't open repo.json for repo {repo_info['id']}") + return return_data + + if debuginfo and not repo_json['with_debuginfo']: + logger.debug(f'Not creating initial repo for {sidetag_name}' + ' as it requires debuginfo and source not') + + sidetag = get_tag(sidetag_id, strict=True) + + # TODO: without event it could be already irrelevant (even if not expired yet) + # maybe at least checking tag_changed_since_event? It would consume some CPU and + # user is maybe not interested in most actual repo (if yes, kojira will still + # regenerate it later) + + logger.debug(f'Copying repo {repo_info["id"]} for sidetag') + new_repo_id, event_id = repo_init(sidetag['name'], with_debuginfo=debuginfo) + + try: + # copy repodata + # TODO: Does it make sense to run RepoDone callbacks? Probably not as it is hard copy. + # koji.plugin.run_callbacks('preRepoDone', repo=repo_info, data=data, expire=False) + dst_repo = koji.pathinfo.repo(new_repo_id, sidetag['name']) + arches = [koji.canonArch(a) for a in koji.parse_arches(sidetag['arches'], to_list=True)] + if repo_json['with_separate_src']: + arches.append('src') + for arch in arches: + src_repodata = f'{src_repo}/{arch}/repodata' + dst_repodata = f'{dst_repo}/{arch}/repodata' + logger.debug(f'Copying repodata {src_repodata} to {dst_repodata}') + if os.path.exists(src_repodata): + # TODO: Could be hard linking dangerous here? + shutil.copytree(src_repodata, dst_repodata, copy_function=os.link) + # mimick host.repoDone, we're sure that there is no previous repo, + # so we can omit most steps + latestrepolink = koji.pathinfo.repo('latest', sidetag['name']) + os.symlink(str(new_repo_id), latestrepolink) + repo_ready(new_repo_id) + # koji.plugin.run_callbacks('postRepoDone', repo=repo_info, data=data, expire=False) + except Exception: + logger.error(f"Copying repo {repo_info['id']} to {new_repo_id} failed.") + repo_problem(new_repo_id) + raise + + return return_data @export From fdedf0936a7336063f6523d3824141d02eed6061 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 13 2023 13:19:14 +0000 Subject: [PATCH 2/4] trigger newRepo task for new sidetag Related: https://pagure.io/koji/issue/3808 --- diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py index 58137c5..a8e16f8 100644 --- a/plugins/cli/sidetag_cli.py +++ b/plugins/cli/sidetag_cli.py @@ -9,8 +9,7 @@ from optparse import OptionParser import koji from koji.plugin import export_cli -from koji_cli.commands import anon_handle_wait_repo -from koji_cli.lib import activate_session, arg_filter +from koji_cli.lib import activate_session, arg_filter, watch_tasks @export_cli @@ -38,7 +37,7 @@ def handle_add_sidetag(options, session, args): if opts.suffix: kwargs['suffix'] = opts.suffix try: - tag = session.createSideTag(basetag, **kwargs) + info = session.createSideTag(basetag, **kwargs) except koji.ActionNotAllowed: parser.error("Policy violation") except koji.ParameterError as ex: @@ -48,13 +47,11 @@ def handle_add_sidetag(options, session, args): raise if not opts.quiet: - print(tag["name"]) + print(info["name"]) if opts.wait: - args = ["--target", tag["name"]] - if opts.quiet: - args.append("--quiet") - anon_handle_wait_repo(options, session, args) + return watch_tasks(session, [info['task_id']], quiet=opts.quiet, + poll_interval=options.poll_interval, topurl=options.topurl) @export_cli diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 9b5e824..4ff1576 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -2,10 +2,6 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import json -import logging -import os -import shutil import koji from koji.context import context from koji.plugin import callback, export @@ -22,19 +18,12 @@ from kojihub import ( get_build_target, get_tag, get_user, + make_task, policy_get_user, readInheritanceData, - repo_init, - repo_problem, - repo_ready, - RootExports, ) from kojihub.db import QueryProcessor, nextval -rootexports = RootExports() - -logger = logging.getLogger('koji.plugin.sidetag') - CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" CONFIG = None ALLOWED_SUFFIXES = [] @@ -98,7 +87,6 @@ def createSideTag(basetag, debuginfo=False, suffix=None): :returns dict: sidetag name + id """ - logger.debug(f'Creating sidetag for {basetag}') if suffix and suffix not in ALLOWED_SUFFIXES: raise koji.GenericError("%s suffix is not allowed for sidetag" % suffix) @@ -148,70 +136,11 @@ def createSideTag(basetag, debuginfo=False, suffix=None): ) _create_build_target(sidetag_name, sidetag_id, sidetag_id) - # Sidetag is ready now, we try to create initial repo but if it is not successful - # kojira will take proper care later - return_data = {"name": sidetag_name, "id": sidetag_id} - - # In case it it is not maven repo, we can copy old one and don't wait for kojira - # for maven and/or src, leave it on kojira - if basetag.get('maven_support'): - # maven repos contains too many files to be handled sync in hub call, let kojira do that - logger.debug(f'Not creating initial repo for {sidetag_name} due to maven_support') - return return_data - - # find active basetag repo - repo_info = rootexports.getRepo(basetag, state=koji.REPO_READY) - if not repo_info: - return return_data - - src_repo = koji.pathinfo.repo(repo_info['id'], basetag['name']) - try: - repo_json = json.load(open(os.path.join(src_repo, 'repo.json'))) - except IOError: - logger.debug("Can't open repo.json for repo {repo_info['id']}") - return return_data + # little higher priority than other newRepo tasks + args = koji.encode_args(sidetag_name, debuginfo=debuginfo) + task_id = make_task('newRepo', args, priority=14, channel='createrepo') - if debuginfo and not repo_json['with_debuginfo']: - logger.debug(f'Not creating initial repo for {sidetag_name}' - ' as it requires debuginfo and source not') - - sidetag = get_tag(sidetag_id, strict=True) - - # TODO: without event it could be already irrelevant (even if not expired yet) - # maybe at least checking tag_changed_since_event? It would consume some CPU and - # user is maybe not interested in most actual repo (if yes, kojira will still - # regenerate it later) - - logger.debug(f'Copying repo {repo_info["id"]} for sidetag') - new_repo_id, event_id = repo_init(sidetag['name'], with_debuginfo=debuginfo) - - try: - # copy repodata - # TODO: Does it make sense to run RepoDone callbacks? Probably not as it is hard copy. - # koji.plugin.run_callbacks('preRepoDone', repo=repo_info, data=data, expire=False) - dst_repo = koji.pathinfo.repo(new_repo_id, sidetag['name']) - arches = [koji.canonArch(a) for a in koji.parse_arches(sidetag['arches'], to_list=True)] - if repo_json['with_separate_src']: - arches.append('src') - for arch in arches: - src_repodata = f'{src_repo}/{arch}/repodata' - dst_repodata = f'{dst_repo}/{arch}/repodata' - logger.debug(f'Copying repodata {src_repodata} to {dst_repodata}') - if os.path.exists(src_repodata): - # TODO: Could be hard linking dangerous here? - shutil.copytree(src_repodata, dst_repodata, copy_function=os.link) - # mimick host.repoDone, we're sure that there is no previous repo, - # so we can omit most steps - latestrepolink = koji.pathinfo.repo('latest', sidetag['name']) - os.symlink(str(new_repo_id), latestrepolink) - repo_ready(new_repo_id) - # koji.plugin.run_callbacks('postRepoDone', repo=repo_info, data=data, expire=False) - except Exception: - logger.error(f"Copying repo {repo_info['id']} to {new_repo_id} failed.") - repo_problem(new_repo_id) - raise - - return return_data + return {"name": sidetag_name, "id": sidetag_id, 'task_id': task_id} @export From f6d67d8478cb6fee7db347e3709e130c5a0fd09a Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 13 2023 13:19:14 +0000 Subject: [PATCH 3/4] make newRepo for sidetag configurable --- diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py index a8e16f8..58137c5 100644 --- a/plugins/cli/sidetag_cli.py +++ b/plugins/cli/sidetag_cli.py @@ -9,7 +9,8 @@ from optparse import OptionParser import koji from koji.plugin import export_cli -from koji_cli.lib import activate_session, arg_filter, watch_tasks +from koji_cli.commands import anon_handle_wait_repo +from koji_cli.lib import activate_session, arg_filter @export_cli @@ -37,7 +38,7 @@ def handle_add_sidetag(options, session, args): if opts.suffix: kwargs['suffix'] = opts.suffix try: - info = session.createSideTag(basetag, **kwargs) + tag = session.createSideTag(basetag, **kwargs) except koji.ActionNotAllowed: parser.error("Policy violation") except koji.ParameterError as ex: @@ -47,11 +48,13 @@ def handle_add_sidetag(options, session, args): raise if not opts.quiet: - print(info["name"]) + print(tag["name"]) if opts.wait: - return watch_tasks(session, [info['task_id']], quiet=opts.quiet, - poll_interval=options.poll_interval, topurl=options.topurl) + args = ["--target", tag["name"]] + if opts.quiet: + args.append("--quiet") + anon_handle_wait_repo(options, session, args) @export_cli diff --git a/plugins/hub/sidetag.conf b/plugins/hub/sidetag.conf index d4e55a4..20a44c4 100644 --- a/plugins/hub/sidetag.conf +++ b/plugins/hub/sidetag.conf @@ -1,9 +1,15 @@ [sidetag] # automatically remove sidetag on untagging last package remove_empty = off + # potential suffixes for sidetag names # allowed_suffixes = + # template for sidetag names. It must contain basetag and tag_id parts # if allowed_suffixes is not empty and suffix was requested, it will be added # as {name_template}-{suffix}. (percent-signs need to be escaped) # name_template = {basetag}-side-{tag_id} + +# Automaticaly trigger newRepo task for every new sidetag. Otherwise let kojira +# prioritize these. +# trigger_new_repo = False diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 4ff1576..8932571 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -27,6 +27,7 @@ from kojihub.db import QueryProcessor, nextval CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" CONFIG = None ALLOWED_SUFFIXES = [] +TRIGGER_NEW_REPO = False def is_sidetag(taginfo, raise_error=False): @@ -136,9 +137,13 @@ def createSideTag(basetag, debuginfo=False, suffix=None): ) _create_build_target(sidetag_name, sidetag_id, sidetag_id) - # little higher priority than other newRepo tasks - args = koji.encode_args(sidetag_name, debuginfo=debuginfo) - task_id = make_task('newRepo', args, priority=14, channel='createrepo') + + if TRIGGER_NEW_REPO: + # little higher priority than other newRepo tasks + args = koji.encode_args(sidetag_name, debuginfo=debuginfo) + task_id = make_task('newRepo', args, priority=14, channel='createrepo') + else: + task_id = None return {"name": sidetag_name, "id": sidetag_id, 'task_id': task_id} @@ -367,3 +372,5 @@ if not CONFIG: NAME_TEMPLATE = CONFIG.get("sidetag", "name_template") else: NAME_TEMPLATE = '{basetag}-side-{tag_id}' + if CONFIG.has_option("sidetag", "trigger_new_repo"): + TRIGGER_NEW_REPO = CONFIG.getboolean("sidetag", "trigger_new_repo") From 7afa72a60d4827235538e908762adcfc8abbeb05 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 13 2023 15:03:23 +0000 Subject: [PATCH 4/4] fix tests --- diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 8932571..6f3a730 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -137,7 +137,6 @@ def createSideTag(basetag, debuginfo=False, suffix=None): ) _create_build_target(sidetag_name, sidetag_id, sidetag_id) - if TRIGGER_NEW_REPO: # little higher priority than other newRepo tasks args = koji.encode_args(sidetag_name, debuginfo=debuginfo) diff --git a/tests/test_plugins/test_sidetag_hub.py b/tests/test_plugins/test_sidetag_hub.py index ebb87dc..8ef301d 100644 --- a/tests/test_plugins/test_sidetag_hub.py +++ b/tests/test_plugins/test_sidetag_hub.py @@ -51,7 +51,7 @@ class TestCreateSideTagHub(unittest.TestCase): self._create_tag.return_value = 12346 ret = sidetag_hub.createSideTag('base_tag') - self.assertEqual(ret, {'name': sidetag_name, 'id': 12346}) + self.assertEqual(ret, {'name': sidetag_name, 'id': 12346, 'task_id': None}) self.get_user.assert_called_once_with(23, strict=True) self.get_tag.assert_called_once_with(self.basetag['name'], strict=True) @@ -83,7 +83,7 @@ class TestCreateSideTagHub(unittest.TestCase): sidetag_hub.NAME_TEMPLATE = '{basetag}-sidetag-{tag_id}' ret = sidetag_hub.createSideTag('base_tag', debuginfo=True, suffix='suffix') - self.assertEqual(ret, {'name': sidetag_name, 'id': 12346}) + self.assertEqual(ret, {'name': sidetag_name, 'id': 12346, 'task_id': None}) def test_createsidetag_template_forbidden_suffix(self): sidetag_hub.ALLOWED_SUFFIXES = ['suffix', 'another']