From 87409499a3f4502d69e76da12421ddb792ce63dc Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:29 +0000 Subject: [PATCH 1/70] draft builds --- diff --git a/builder/kojid b/builder/kojid index b4e72b5..3178fd3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1102,6 +1102,8 @@ class BuildTask(BaseTaskHandler): % (data['name'], target_info['dest_tag_name'])) # TODO - more pre tests archlist = self.getArchList(build_tag, h, extra=extra_arches) + # pass draft option in + data['draft'] = opts.get('draft') # let the system know about the build we're attempting if not self.opts.get('scratch'): # scratch builds do not get imported @@ -2176,6 +2178,8 @@ class WrapperRPMTask(BaseBuildTask): data['extra'] = {'source': {'original_url': source['url']}} if opts.get('custom_user_metadata'): data['extra']['custom_user_metadata'] = opts['custom_user_metadata'] + # pass draft option in + data['draft'] = opts.get('draft') self.logger.info("Reading package config for %(name)s" % data) pkg_cfg = self.session.getPackageConfig(build_target['dest_tag'], data['name']) if not opts.get('skip_tag'): @@ -5249,6 +5253,7 @@ Subject: %(nvr)s %(result)s %(operation)s by %(user_name)s\r To: %(to_addrs)s\r X-Koji-Package: %(pkg_name)s\r X-Koji-NVR: %(nvr)s\r +X-Koji-Draft: %(draft)s\r X-Koji-User: %(user_name)s\r X-Koji-Status: %(status)s\r %(tag_headers)s\r @@ -5278,6 +5283,7 @@ Status: %(status)s\r user = self.session.getUser(user_info) pkg_name = build['package_name'] nvr = koji.buildLabel(build) + draft = build.get('draft', False) user_name = user['name'] from_addr = self.options.from_addr @@ -5349,6 +5355,7 @@ X-Koji-Tag: %(dest_tag)s\r X-Koji-Package: %(build_pkg_name)s\r X-Koji-Builder: %(build_owner)s\r X-Koji-Status: %(status)s\r +X-Koji-Draft: %(draft)s\r \r Package: %(build_nvr)s\r Tag: %(dest_tag)s\r @@ -5448,6 +5455,7 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r build_nvr = koji.buildLabel(build) build_id = build['id'] build_owner = build['owner_name'] + draft = build.get('draft', False) # target comes from session.py:_get_build_target() dest_tag = None if target is not None: diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 9c52358..5da1770 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -580,6 +580,8 @@ def handle_build(options, session, args): parser.add_option("--custom-user-metadata", type="str", help="Provide a JSON string of custom metadata to be deserialized and " "stored under the build's extra.custom_user_metadata field") + parser.add_option("--draft", action="store_true", + help="Build draft build instead") (build_opts, args) = parser.parse_args(args) if len(args) != 2: parser.error("Exactly two arguments (a build target and a SCM URL or srpm file) are " @@ -588,6 +590,8 @@ def handle_build(options, session, args): parser.error("--no-/rebuild-srpm is only allowed for --scratch builds") if build_opts.arch_override and not build_opts.scratch: parser.error("--arch_override is only allowed for --scratch builds") + if build_opts.scratch and build_opts.draft: + parser.error("--scratch and --draft cannot be both specfied") custom_user_metadata = {} if build_opts.custom_user_metadata: try: @@ -618,7 +622,7 @@ def handle_build(options, session, args): if build_opts.arch_override: opts['arch_override'] = koji.parse_arches(build_opts.arch_override) for key in ('skip_tag', 'scratch', 'repo_id', 'fail_fast', 'wait_repo', 'wait_builds', - 'rebuild_srpm'): + 'rebuild_srpm', 'draft'): val = getattr(build_opts, key) if val is not None: opts[key] = val @@ -830,6 +834,8 @@ def handle_wrapper_rpm(options, session, args): parser.add_option("--nowait", action="store_false", dest="wait", help="Don't wait on build") parser.add_option("--background", action="store_true", help="Run the build at a lower priority") + parser.add_option("--draft", action="store_true", + help="Build draft build instead") (build_opts, args) = parser.parse_args(args) if build_opts.inis: @@ -839,6 +845,8 @@ def handle_wrapper_rpm(options, session, args): if len(args) < 3: parser.error("You must provide a build target, a build ID or NVR, " "and a SCM URL to a specfile fragment") + if build_opts.scratch and build_opts.draft: + parser.error("--scratch and --draft cannot be both specfied") activate_session(session, options) target = args[0] @@ -874,6 +882,8 @@ def handle_wrapper_rpm(options, session, args): opts['skip_tag'] = True if build_opts.scratch: opts['scratch'] = True + if build_opts.draft: + opts['draft'] = True task_id = session.wrapperRPM(build_id, url, target, priority, opts=opts) print("Created task: %d" % task_id) print("Task info: %s/taskinfo?taskID=%s" % (options.weburl, task_id)) @@ -1314,6 +1324,10 @@ def handle_import(goptions, session, args): parser.add_option("--test", action="store_true", help="Don't actually import") parser.add_option("--create-build", action="store_true", help="Auto-create builds as needed") parser.add_option("--src-epoch", help="When auto-creating builds, use this epoch") + parser.add_option("--create-draft", action="store_true", + help="Auto-create draft builds instead as needed") + parser.add_option("--draft-build", metavar='NVR|ID', + help="The target draft build to import to") (options, args) = parser.parse_args(args) if len(args) < 1: parser.error("At least one package must be specified") @@ -1324,6 +1338,35 @@ def handle_import(goptions, session, args): options.src_epoch = int(options.src_epoch) except (ValueError, TypeError): parser.error("Invalid value for epoch: %s" % options.src_epoch) + if options.create_draft: + print("Will create draft build instead if desired nvr doesn't exist") + options.create_build = True + draft_build = None + draft_target_nvr = None + if options.draft_build: + if options.create_build: + parser.error( + "To ensure no misuse, don't specify --draft-build while auto-creating." + ) + try: + draft_build = int(options.draft_build) + except ValueError: + draft_build = options.draft_build + draft_build = session.getBuild(draft_build) + if not draft_build: + error("No such build: %s" % options.draft_build) + if not draft_build.get('draft'): + error("%s is not a draft build" % draft_build['nvr']) + b_state = koji.BUILD_STATES[draft_build['state']] + if b_state != 'COMPLETE': + error("draft build %s is expected as COMPLETE, got %s" % (draft_build['nvr'], b_state)) + target_release = draft_build.get('extra', {}).get('draft', {}).get('target_release') + if not target_release: + error("Invalid draft build: %s," + " no draft.target_release found in extra" % draft_build['nvr']) + draft_target_nvr = "%s-%s-%s" % ( + draft_build['name'], draft_build['version'], target_release + ) activate_session(session, goptions) to_import = {} for path in args: @@ -1335,6 +1378,7 @@ def handle_import(goptions, session, args): else: nvr = "%(name)s-%(version)s-%(release)s" % koji.parse_NVRA(data['sourcerpm']) to_import.setdefault(nvr, []).append((path, data)) + builds_missing = False nvrs = sorted(to_list(to_import.keys())) for nvr in nvrs: @@ -1343,29 +1387,41 @@ def handle_import(goptions, session, args): if data['sourcepackage']: break else: + if nvr == draft_target_nvr: + continue + # when no srpm and create_draft + elif options.create_draft: + print("Missing srpm for draft build creating with target nvr: %s" % nvr) + builds_missing = True + continue # no srpm included, check for build binfo = session.getBuild(nvr) if not binfo: print("Missing build or srpm: %s" % nvr) builds_missing = True - if builds_missing and not options.create_build: + if builds_missing and (not options.create_build or options.create_draft): print("Aborting import") return # local function to help us out below - def do_import(path, data): + def do_import(path, data, draft_build=None): + draft = bool(draft_build) or options.create_draft rinfo = dict([(k, data[k]) for k in ('name', 'version', 'release', 'arch')]) - prev = session.getRPM(rinfo) - if prev and not prev.get('external_repo_id', 0): - if prev['payloadhash'] == koji.hex_string(data['sigmd5']): - print("RPM already imported: %s" % path) - else: - warn("md5sum mismatch for %s" % path) - warn(" A different rpm with the same name has already been imported") - warn(" Existing sigmd5 is %r, your import has %r" % ( - prev['payloadhash'], koji.hex_string(data['sigmd5']))) - print("Skipping import") - return + if draft_build or not options.create_draft: + opts = {} + if draft_build: + opts['build'] = draft_build + prev = session.getRPM(rinfo, **opts) + if prev and not prev.get('external_repo_id', 0): + if prev['payloadhash'] == koji.hex_string(data['sigmd5']): + print("RPM already imported: %s" % path) + else: + warn("md5sum mismatch for %s" % path) + warn(" A different rpm with the same name has already been imported") + warn(" Existing sigmd5 is %r, your import has %r" % ( + prev['payloadhash'], koji.hex_string(data['sigmd5']))) + print("Skipping import") + return if options.test: print("Test mode -- skipping import for %s" % path) return @@ -1381,41 +1437,69 @@ def handle_import(goptions, session, args): sys.stdout.write("importing %s... " % path) sys.stdout.flush() try: - session.importRPM(serverdir, os.path.basename(path)) + opts = {} + if draft_build: + opts['build'] = draft_build + if draft: + opts['draft'] = draft + rpminfo = session.importRPM(serverdir, os.path.basename(path), **opts) except koji.GenericError as e: + rpminfo = None print("\nError importing: %s" % str(e).splitlines()[-1]) sys.stdout.flush() else: print("done") sys.stdout.flush() + return rpminfo for nvr in nvrs: # check for existing build need_build = True - binfo = session.getBuild(nvr) - if binfo: - b_state = koji.BUILD_STATES[binfo['state']] - if b_state == 'COMPLETE': - need_build = False - elif b_state in ['FAILED', 'CANCELED']: - if not options.create_build: - print("Build %s state is %s. Skipping import" % (nvr, b_state)) + is_draft = True + if nvr == draft_target_nvr: + binfo = draft_build + need_build = False + elif options.create_draft: + binfo = None + need_build = True + else: + is_draft = False + binfo = session.getBuild(nvr) + if binfo: + b_state = koji.BUILD_STATES[binfo['state']] + if b_state == 'COMPLETE': + need_build = False + elif b_state in ['FAILED', 'CANCELED']: + if not options.create_build: + print("Build %s state is %s. Skipping import" % (nvr, b_state)) + continue + else: + print("Build %s exists with state=%s. Skipping import" % (nvr, b_state)) continue - else: - print("Build %s exists with state=%s. Skipping import" % (nvr, b_state)) - continue # import srpms first, if any for path, data in to_import[nvr]: if data['sourcepackage']: - if binfo and b_state != 'COMPLETE': + # we can not fix state for draft build by createEmptyBuild + if not is_draft and binfo and b_state != 'COMPLETE': # need to fix the state print("Creating empty build: %s" % nvr) b_data = koji.util.dslice(binfo, ['name', 'version', 'release']) b_data['epoch'] = data['epoch'] session.createEmptyBuild(**b_data) binfo = session.getBuild(nvr) - do_import(path, data) + dbld = binfo if is_draft else None + will_create = False + if options.create_draft and not dbld: + will_create = True + print("Will create draft build with target nvr: %s while importing" % nvr) + rpminfo = do_import(path, data, draft_build=dbld) + # only needed for draft build, but + # TODO: should be able to apply to regular import + if rpminfo and rpminfo.get('build', {}).get('draft'): + binfo = rpminfo['build'] + if will_create: + print("Draft build: %s created" % binfo['nvr']) need_build = False if need_build: @@ -1424,11 +1508,11 @@ def handle_import(goptions, session, args): if binfo: # should have caught this earlier, but just in case... b_state = koji.BUILD_STATES[binfo['state']] - print("Build %s state is %s. Skipping import" % (nvr, b_state)) + print("Build %s state is %s. Skipping import" % (binfo['nvr'], b_state)) continue else: print("No such build: %s (include matching srpm or use " - "--create-build option to add it)" % nvr) + "--create-build/--create-draft option to add it)" % nvr) continue else: # let's make a new build @@ -1439,17 +1523,22 @@ def handle_import(goptions, session, args): # pull epoch from first rpm data = to_import[nvr][0][1] b_data['epoch'] = data['epoch'] + if options.create_draft: + b_data['draft'] = True + b_display = "empty draft build with target nvr: %s" % nvr + else: + b_display = "empty build: %s" % nvr if options.test: - print("Test mode -- would have created empty build: %s" % nvr) + print("Test mode -- would have created %s" % b_display) else: - print("Creating empty build: %s" % nvr) - session.createEmptyBuild(**b_data) - binfo = session.getBuild(nvr) + print("Creating %s" % b_display) + buildid = session.createEmptyBuild(**b_data) + binfo = session.getBuild(buildid) for path, data in to_import[nvr]: if data['sourcepackage']: continue - do_import(path, data) + do_import(path, data, draft_build=binfo if is_draft else None) def handle_import_cg(goptions, session, args): @@ -2727,11 +2816,15 @@ def anon_handle_list_tagged(goptions, session, args): parser.add_option("--ts", type='int', metavar="TIMESTAMP", help="query at last event before timestamp") parser.add_option("--repo", type='int', metavar="REPO#", help="query at event for a repo") + parser.add_option("--draft-only", action="store_true", help="Only list draft builds/rpms") + parser.add_option("--no-draft", action="store_true", help="Only list regular builds/rpms") (options, args) = parser.parse_args(args) if len(args) == 0: parser.error("A tag name must be specified") elif len(args) > 2: parser.error("Only one package name may be specified") + if options.no_draft and options.draft_only: + parser.error("--draft-only conflicts with --no-draft") ensure_connection(session, goptions) pathinfo = koji.PathInfo() package = None @@ -2753,6 +2846,10 @@ def anon_handle_list_tagged(goptions, session, args): options.rpms = True if options.type: opts['type'] = options.type + elif options.no_draft: + opts['draft'] = koji.FLAG_REGULAR_BUILD + elif options.draft_only: + opts['draft'] = koji.FLAG_DRAFT_BUILD event = koji.util.eventFromOpts(session, options) event_id = None if event: @@ -2798,7 +2895,9 @@ def anon_handle_list_tagged(goptions, session, args): fmt = "%(path)s" data = [x for x in data if 'path' in x] else: - fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" + fmt = "%(name)s-%(version)s-%(release)s.%(arch)s%(draft_suffix)s" + for x in data: + x['draft_suffix'] = (' (#draft_%s)' % x['build_id']) if x.get('draft') else '' if options.sigs: fmt = "%(sigkey)s " + fmt else: @@ -2861,10 +2960,13 @@ def anon_handle_list_buildroot(goptions, session, args): fmt = "%(nvr)s.%(arch)s" order = sorted([(fmt % x, x) for x in list_rpms]) for nvra, rinfo in order: - if options.verbose and rinfo.get('is_update'): - print("%s [update]" % nvra) - else: - print(nvra) + line = nvra + if options.verbose: + if rinfo.get('draft'): + line += " (#draft_%s)" % rinfo['build_id'] + if rinfo.get('is_update'): + line += " [update]" + print(line) list_archives = session.listArchives(**opts) if list_archives: @@ -3403,6 +3505,8 @@ def anon_handle_list_builds(goptions, session, args): parser.add_option("--source", help="Only builds where the source field matches (glob pattern)") parser.add_option("--owner", help="List builds built by this owner") parser.add_option("--volume", help="List builds by volume ID") + parser.add_option("--draft-only", action="store_true", help="Only list draft builds") + parser.add_option("--no-draft", action="store_true", help="Only list regular builds") parser.add_option("-k", "--sort-key", action="append", metavar='FIELD', default=[], help="Sort the list by the named field. Allowed sort keys: " "build_id, owner_name, state") @@ -3419,6 +3523,12 @@ def anon_handle_list_builds(goptions, session, args): value = getattr(options, key) if value is not None: opts[key] = value + if options.no_draft and options.draft_only: + parser.error("--draft-only conflits with --no-draft") + elif options.no_draft: + opts['draft'] = koji.FLAG_REGULAR_BUILD + elif options.draft_only: + opts['draft'] = koji.FLAG_DRAFT_BUILD if options.cg: opts['cgID'] = options.cg if options.package: @@ -3520,13 +3630,24 @@ def anon_handle_rpminfo(goptions, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--buildroots", action="store_true", help="show buildroots the rpm was used in") + parser.add_option("--build", metavar="NVR|ID", + help="show the rpm(s) in the build") + (options, args) = parser.parse_args(args) if len(args) < 1: parser.error("Please specify an RPM") + opts = {} + build = options.build + if options.build: + try: + build = int(build) + except ValueError: + pass + opts['build'] = build ensure_connection(session, goptions) error_hit = False for rpm in args: - info = session.getRPM(rpm) + info = session.getRPM(rpm, **opts) if info is None: warn("No such rpm: %s\n" % rpm) error_hit = True @@ -3536,24 +3657,29 @@ def anon_handle_rpminfo(goptions, session, args): else: info['epoch'] = str(info['epoch']) + ":" if not info.get('external_repo_id', 0): - buildinfo = session.getBuild(info['build_id']) - buildinfo['name'] = buildinfo['package_name'] - buildinfo['arch'] = 'src' - if buildinfo['epoch'] is None: - buildinfo['epoch'] = "" + if info['arch'] == 'src': + srpminfo = info.copy() else: - buildinfo['epoch'] = str(buildinfo['epoch']) + ":" + srpminfo = session.listRPMs(buildID=info['build_id'], arches='src')[0] + if srpminfo['epoch'] is None: + srpminfo['epoch'] = "" + else: + srpminfo['epoch'] = str(srpminfo['epoch']) + ":" + buildinfo = session.getBuild(info['build_id']) print("RPM: %(epoch)s%(name)s-%(version)s-%(release)s.%(arch)s [%(id)d]" % info) + if info.get('draft'): + print("Draft: YES") if info.get('external_repo_id'): repo = session.getExternalRepo(info['external_repo_id']) print("External Repository: %(name)s [%(id)i]" % repo) print("External Repository url: %(url)s" % repo) else: + print("Build: %(nvr)s [%(id)d]" % buildinfo) print("RPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info))) - print("SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo) + print("SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % srpminfo) print("SRPM Path: %s" % - os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(buildinfo))) + os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(srpminfo))) print("Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(info['buildtime']))) print("SIGMD5: %(payloadhash)s" % info) @@ -3563,6 +3689,7 @@ def anon_handle_rpminfo(goptions, session, args): headers=["license"]) if 'license' in headers: print("License: %(license)s" % headers) + # kept for backward compatibility print("Build ID: %(build_id)s" % info) if info['buildroot_id'] is None: print("No buildroot data available") @@ -3619,6 +3746,8 @@ def anon_handle_buildinfo(goptions, session, args): info['arch'] = 'src' info['state'] = koji.BUILD_STATES[info['state']] print("BUILD: %(name)s-%(version)s-%(release)s [%(id)d]" % info) + if info.get('draft'): + print("Draft: YES") print("State: %(state)s" % info) if info['state'] == 'BUILDING': print("Reserved by: %(cg_name)s" % info) diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index c605940..b3a0812 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -925,3 +925,10 @@ class DatetimeJSONEncoder(json.JSONEncoder): if isinstance(o, xmlrpc_client.DateTime): return str(o) return json.JSONEncoder.default(self, o) + + +def yesno(x): + if x: + return 'Y' + else: + return 'N' diff --git a/koji/__init__.py b/koji/__init__.py index 11f6cba..b0b372e 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -27,6 +27,7 @@ from __future__ import absolute_import, division import base64 import datetime import errno +import functools import hashlib import json import logging @@ -274,6 +275,7 @@ TAG_UPDATE_TYPES = Enum(( 'VOLUME_CHANGE', 'IMPORT', 'MANUAL', + 'DRAFT_PROMOTION', )) # BEGIN kojikamid dup # @@ -295,6 +297,44 @@ PRIO_DEFAULT = 20 DEFAULT_REQUEST_TIMEOUT = 60 * 60 * 12 DEFAULT_AUTH_TIMEOUT = 60 +# draft release format +DRAFT_RELEASE_FORMAT = '{release}#draft_{id}' + +FLAG_DRAFT_BUILD = 1 +FLAG_REGULAR_BUILD = 2 +FLAG_ALL_BUILD = FLAG_DRAFT_BUILD | FLAG_REGULAR_BUILD + +if six.PY3: + from enum import IntFlag + + # draft build bit FLAGs + class DRAFT_FLAG(IntFlag): + + DRAFT = FLAG_DRAFT_BUILD + REGULAR = FLAG_REGULAR_BUILD + ALL = FLAG_ALL_BUILD + + @classmethod + def _missing_(cls, value): + if not isinstance(value, int): + raise ValueError("%r is not a valid %s" % (value, cls.__name__)) + # diable member creation and negative integer + return cls._value2member_map_.get(value) + + def convert_draft_option(func=None, kw='draft'): + def wrapper(func): + @functools.wraps(func) + def convert(*args, **kwargs): + if kw in kwargs: + kwargs[kw] = DRAFT_FLAG(kwargs[kw]) + return func(*args, **kwargs) + return convert + if func is None: + return wrapper + else: + return wrapper(func) + + # BEGIN kojikamid dup # # Exceptions @@ -2363,6 +2403,39 @@ class PathInfo(object): return self.volumedir(build.get('volume_name')) + \ ("/packages/%(name)s/%(version)s/%(release)s" % build) + def to_buildinfo(self, path): + """Revert build dir path (/[/]packages///) + to build map with the data below: + + - name + - version + - release + - volume_name + """ + path = path.rstrip('/') + parts = path.rsplit('/', 4) + if len(parts) != 5: + raise GenericError("Invalid build path: %s" % path) + if parts[-4] != 'packages': + raise GenericError("Invalid build path: %s, packages subdir not found" % path) + nvr = parts[-3:] + rest = parts[0] + if not rest.startswith(self.topdir): + raise GenericError("Invalid build path: %s, topdir is not %r" % (path, self.topdir)) + vol_part = rest[len(self.topdir):] + if not vol_part: + vol = 'DEFAULT' + elif vol_part.startswith('/vol/'): + vol = vol_part[5:] + else: + raise GenericError( + "Invalid build path: %s, volume dir: %s is incorrect" % (path, vol_part) + ) + return {'name': nvr[0], + 'version': nvr[1], + 'release': nvr[2], + 'volume_name': vol} + def mavenbuild(self, build): """Return the directory where the Maven build exists in the global store (/mnt/koji/packages)""" @@ -2410,6 +2483,27 @@ class PathInfo(object): """Return the path (relative to build_dir) where an rpm belongs""" return "%(arch)s/%(name)s-%(version)s-%(release)s.%(arch)s.rpm" % rpminfo + def to_rpminfo(self, path, full=False): + """Revert rpm path in build dir to rpm nvra map with/without build dir""" + if path.endswith('/'): + raise GenericError("Invalid path: %s, cannot be a directory" % path) + parts = path.rsplit('/', 2) + if full: + if len(parts) != 3: + raise GenericError("No build dir in path: %s" % path) + return self.to_buildinfo(parts[-3]), self._to_rpminfo(parts[-2:]) + return self._to_rpminfo(parts) + + def _to_rpminfo(self, parts): + if len(parts) != 2 or not parts[0]: + raise GenericError("Invalid path: %s" % '/'.join(parts)) + rpminfo = parse_NVRA(parts[-1]) + if parts[0] != rpminfo['arch']: + raise GenericError( + 'mismatch between arch dir (%s) and arch (%s) in rpm' % (parts[0], rpminfo) + ) + return rpminfo + def signed(self, rpminfo, sigkey): """Return the path (relative to build dir) where a signed rpm lives""" return "data/signed/%s/" % sigkey + self.rpm(rpminfo) diff --git a/koji/plugin.py b/koji/plugin.py index 6306e97..89c0863 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -60,6 +60,8 @@ callbacks = { 'postRepoInit': [], 'preRepoDone': [], 'postRepoDone': [], + 'preBuildPromote': [], + 'postBuildPromote': [], 'preCommit': [], 'postCommit': [], # builder diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 79ae413..81d71e2 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -55,6 +55,7 @@ import rpm from psycopg2._psycopg import IntegrityError import koji +from koji import DRAFT_FLAG, convert_draft_option import koji.plugin import koji.policy import koji.rpmdiff @@ -68,6 +69,7 @@ from koji.util import ( base64encode, decode_bytes, dslice, + encode_datetime, extract_build_task, joinpath, md5_constructor, @@ -1393,8 +1395,9 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None, pattern=None return query.iterate() +@convert_draft_option def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, - type=None, extra=False): + type=None, extra=False, *, draft=DRAFT_FLAG.ALL): """Returns a list of builds for specified tag :param int tag: tag ID @@ -1406,6 +1409,10 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, :param str type: restrict the list to builds of the given type. Currently the supported types are 'maven', 'win', 'image', or any custom content generator btypes. :param bool extra: Set to "True" to get the build extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns [dict]: list of buildinfo dicts """ # build - id pkg_id version release epoch @@ -1432,6 +1439,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), + ('build.draft', 'draft'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), @@ -1477,6 +1485,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, clauses.append('package.name = %(package)s') if owner: clauses.append('users.name = %(owner)s') + append_draft_clause(draft, clauses) queryOpts = {'order': '-create_event'} # latest first if extra: fields.append(('build.extra', 'extra')) @@ -1513,8 +1522,9 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, return builds +@convert_draft_option def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, latest=True, - rpmsigs=False, owner=None, type=None, extra=True): + rpmsigs=False, owner=None, type=None, extra=True, *, draft=DRAFT_FLAG.ALL): """Returns a list of rpms and builds for specified tag :param int|str tag: The tag name or ID to search @@ -1534,6 +1544,10 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late :param str type: Filter by build type. Supported types are 'maven', 'win', and 'image'. :param bool extra: Set to "False" to skip the rpm extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns: a two-element list. The first element is the list of RPMs, and the second element is the list of builds. """ @@ -1544,7 +1558,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, - owner=owner, type=type) + owner=owner, type=type, draft=draft) # index builds build_idx = dict([(b['build_id'], b) for b in builds]) @@ -1555,6 +1569,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late ('rpminfo.arch', 'arch'), ('rpminfo.id', 'id'), ('rpminfo.epoch', 'epoch'), + ('rpminfo.draft', 'draft'), ('rpminfo.payloadhash', 'payloadhash'), ('rpminfo.size', 'size'), ('rpminfo.buildtime', 'buildtime'), @@ -1582,6 +1597,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late clauses.append('rpminfo.arch IN %(arch)s') else: raise koji.GenericError('Invalid type for arch option: %s' % builtins.type(arch)) + append_draft_clause(draft, clauses, table='rpminfo') if extra: fields.append(('rpminfo.extra', 'extra')) @@ -4357,6 +4373,7 @@ def get_build(buildInfo, strict=False): release epoch nvr + draft: Whether the build is draft or not state task_id: ID of the task that kicked off the build owner_id: ID of the user who kicked off the build @@ -4392,7 +4409,9 @@ def get_build(buildInfo, strict=False): fields = (('build.id', 'id'), ('build.version', 'version'), ('build.release', 'release'), ('build.id', 'build_id'), - ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.epoch', 'epoch'), + ('build.draft', 'draft'), + ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), @@ -4481,6 +4500,8 @@ def get_next_release(build_info, incr=1): This method searches the latest building, successful, or deleted build and returns the "next" release value for that version. + Note that draft builds are excluded while getting that latest build. + Examples: None becomes "1" @@ -4512,7 +4533,7 @@ def get_next_release(build_info, incr=1): query = QueryProcessor(tables=['build'], joins=['package ON build.pkg_id = package.id'], columns=['build.id', 'release'], clauses=['name = %(name)s', 'version = %(version)s', - 'state in %(states)s'], + 'state in %(states)s', 'NOT draft'], values=values, opts={'order': '-build.id', 'limit': 1}) result = query.executeOne() @@ -4575,10 +4596,10 @@ def _fix_rpm_row(row): _fix_archive_row = _fix_rpm_row -def get_rpm(rpminfo, strict=False, multi=False): +def get_rpm(rpminfo, strict=False, multi=False, build=None): """Get information about the specified RPM - rpminfo may be any one of the following: + rpminfo ma4666y be any one of the following: - a int ID - a string N-V-R.A - a string N-V-R.A@location @@ -4587,12 +4608,17 @@ def get_rpm(rpminfo, strict=False, multi=False): If specified, location should match the name of an external repo + If build is specfied, the rpm is limited to the build's + + build and location(not INTERNAL) is conflict because a rpm in + A map will be returned, with the following keys: - id - name - version - release - arch + - draft - epoch - payloadhash - size @@ -4620,6 +4646,7 @@ def get_rpm(rpminfo, strict=False, multi=False): ('release', 'release'), ('epoch', 'epoch'), ('arch', 'arch'), + ('draft', 'draft'), ('external_repo_id', 'external_repo_id'), ('external_repo.name', 'external_repo_name'), ('payloadhash', 'payloadhash'), @@ -4644,6 +4671,12 @@ def get_rpm(rpminfo, strict=False, multi=False): else: clauses.append("rpminfo.name=%(name)s AND version=%(version)s " "AND release=%(release)s AND arch=%(arch)s") + # build and non-INTERNAL location (and multi=True) conflict in theory, + # but we just do the query and return None. + if build: + # strict=True as we treate build not found as an input error + data['build_id'] = find_build_id(build, strict=True) + clauses.append("rpminfo.build_id = %(build_id)s") retry = False if 'location' in data: data['external_repo_id'] = get_external_repo_id(data['location'], strict=True) @@ -4675,8 +4708,9 @@ def get_rpm(rpminfo, strict=False, multi=False): return ret +@convert_draft_option def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, - arches=None, queryOpts=None): + arches=None, queryOpts=None, *, draft=DRAFT_FLAG.ALL): """List RPMS. If buildID, imageID and/or buildrootID are specified, restrict the list of RPMs to only those RPMs that are part of that build, or were built in that buildroot. If componentBuildrootID is specified, @@ -4691,6 +4725,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID - nvr (synthesized for sorting purposes) - arch - epoch + - draft - payloadhash - size - buildtime @@ -4706,12 +4741,22 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID - is_update If no build has the given ID, or the build generated no RPMs, - an empty list is returned.""" + an empty list is returned. + + The option draft with a bit flag(enum.IntFlag) value is to filter rpm by that + rpm belongs to a draft build, a regular build or both (default). + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular + """ + fields = [('rpminfo.id', 'id'), ('rpminfo.name', 'name'), ('rpminfo.version', 'version'), ('rpminfo.release', 'release'), ("rpminfo.name || '-' || rpminfo.version || '-' || rpminfo.release", 'nvr'), ('rpminfo.arch', 'arch'), - ('rpminfo.epoch', 'epoch'), ('rpminfo.payloadhash', 'payloadhash'), + ('rpminfo.epoch', 'epoch'), + ('rpminfo.draft', 'draft'), + ('rpminfo.payloadhash', 'payloadhash'), ('rpminfo.size', 'size'), ('rpminfo.buildtime', 'buildtime'), ('rpminfo.build_id', 'build_id'), ('rpminfo.buildroot_id', 'buildroot_id'), ('rpminfo.external_repo_id', 'external_repo_id'), @@ -4749,6 +4794,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID clauses.append('rpminfo.arch = %(arches)s') else: raise koji.GenericError('Invalid type for "arches" parameter: %s' % type(arches)) + append_draft_clause(draft, clauses) fields, aliases = zip(*fields) query = QueryProcessor(columns=fields, aliases=aliases, @@ -5959,7 +6005,7 @@ def check_volume_policy(data, strict=False, default=None): return None -def apply_volume_policy(build, strict=False): +def apply_volume_policy(build, strict=False, dry_run=False): """Apply volume policy, moving build as needed build should be the buildinfo returned by get_build() @@ -5967,12 +6013,16 @@ def apply_volume_policy(build, strict=False): The strict options determines what happens in the case of a bad policy. If strict is True, an exception will be raised. Otherwise, the existing volume we be retained. + + If dry_run is True, return the volume instead of doing the actual moving. """ policy_data = {'build': build} task_id = extract_build_task(build) if task_id: policy_data.update(policy_data_from_task(task_id)) volume = check_volume_policy(policy_data, strict=strict) + if dry_run: + return volume if volume is None: # just leave the build where it is return @@ -5985,6 +6035,10 @@ def apply_volume_policy(build, strict=False): def new_build(data, strict=False): """insert a new build entry + If the build to create is a draft, the release field is the target release + rather than its actual release with draft suffix. The draft suffix will be + generated here as #draft_. + If strict is specified, raise an exception, if build already exists. """ @@ -6005,8 +6059,11 @@ def new_build(data, strict=False): for f in ('version', 'release', 'epoch'): if f not in data: raise koji.GenericError("No %s value for build" % f) + extra = {} if 'extra' in data: try: + extra = data['extra'] + # backwards compatible for data in callback data['extra'] = json.dumps(data['extra']) except Exception: raise koji.GenericError("No such build extra data: %(extra)r" % data) @@ -6021,9 +6078,12 @@ def new_build(data, strict=False): data.setdefault('owner', context.session.user_id) data.setdefault('task_id', None) data.setdefault('volume_id', 0) + data.setdefault('draft', False) - # check for existing build - old_binfo = get_build(data) + old_binfo = None + if not data.get('draft'): + # check for existing build + old_binfo = get_build(data) if old_binfo: if strict: raise koji.GenericError(f'Existing build found: {old_binfo}') @@ -6034,12 +6094,30 @@ def new_build(data, strict=False): new=data['state'], info=data) # insert the new data - insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', - 'task_id', 'owner', 'start_time', 'completion_time', 'source', - 'extra']) + insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'draft', 'state', + 'volume_id', 'task_id', 'owner', 'start_time', 'completion_time', + 'source', 'extra']) if 'cg_id' in data: insert_data['cg_id'] = data['cg_id'] data['id'] = insert_data['id'] = nextval('build_id_seq') + # handle draft suffix in release + if data.get('draft'): + target_release = data['release'] + data['release'] = insert_data['release'] = koji.DRAFT_RELEASE_FORMAT.format(**data) + # it's still possible to already have a build with the same nvr + draft_nvr = dslice(data, ['name', 'version', 'release']) + if find_build_id(draft_nvr): + raise koji.GenericError( + f"The build already exists: {draft_nvr}" + ) + # data.extra.draft should not contains anything + if extra.get('draft'): + raise koji.GenericError('"draft" found in input extra which is reserved already') + extra['draft'] = { + 'target_release': target_release, + 'promoted': False + } + insert_data['extra'] = json.dumps(extra) or None insert = InsertProcessor('build', data=insert_data) insert.execute() new_binfo = get_build(data['id'], strict=True) @@ -6157,7 +6235,7 @@ def check_noarch_rpms(basepath, rpms, logs=None): return result -def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None): +def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, draft=False): """Import a build into the database (single transaction) Files must be uploaded and specified with path relative to the workdir @@ -6167,6 +6245,9 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) brmap - dictionary mapping [s]rpms to buildroot ids task_id - associate the build with a task build_id - build is a finalization of existing entry + draft - If True and + - build_id: None, to create a draft build + Not None, the build_id must be a draft build with a valid release """ if brmap is None: brmap = {} @@ -6217,16 +6298,20 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) build['volume_name'] = vol['name'] if build_id is None: + build['draft'] = draft build_id = new_build(build) binfo = get_build(build_id, strict=True) new_typed_build(binfo, 'rpm') else: # build_id was passed in - sanity check + build['id'] = build_id binfo = get_build(build_id, strict=True) st_complete = koji.BUILD_STATES['COMPLETE'] st_old = binfo['state'] koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=binfo) + if draft: + build['release'] = koji.DRAFT_RELEASE_FORMAT.format(**build) for key in ('name', 'version', 'release', 'epoch', 'task_id'): if build[key] != binfo[key]: raise koji.GenericError( @@ -6266,11 +6351,17 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) return binfo -def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): +def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, draft=False): """Import a single rpm into the database Designed to be called from import_build. """ + # draft option must be sane + if buildinfo and draft != buildinfo.get('draft', False): + raise koji.GenericError( + f'draft property: {buildinfo["draft"]} of build: {buildinfo["id"]} mismatch,' + f' {draft} is expected' + ) if not os.path.exists(fn): raise koji.GenericError("No such file: %s" % fn) @@ -6278,6 +6369,7 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): hdr = koji.get_rpm_header(fn) rpminfo = koji.get_header_fields(hdr, ['name', 'version', 'release', 'epoch', 'sourcepackage', 'arch', 'buildtime', 'sourcerpm']) + rpminfo['draft'] = draft if rpminfo['sourcepackage'] == 1: rpminfo['arch'] = "src" @@ -6290,26 +6382,37 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): if buildinfo is None: # figure it out for ourselves if rpminfo['sourcepackage'] == 1: - buildinfo = get_build(rpminfo, strict=False) + if not draft: + # we only check if nvr already exists for non-draft + buildinfo = get_build(rpminfo, strict=False) if not buildinfo: # create a new build build_id = new_build(rpminfo) # we add the rpm build type below buildinfo = get_build(build_id, strict=True) else: + if draft: + raise koji.GenericError( + f'Cannot import draft rpm: {basename} without specifying a build' + ) # figure it out from sourcerpm string buildinfo = get_build(koji.parse_NVRA(rpminfo['sourcerpm'])) if buildinfo is None: # XXX - handle case where package is not a source rpm # and we still need to create a new build raise koji.GenericError('No such build') - state = koji.BUILD_STATES[buildinfo['state']] - if state in ('FAILED', 'CANCELED', 'DELETED'): - nvr = "%(name)s-%(version)s-%(release)s" % buildinfo - raise koji.GenericError("Build is %s: %s" % (state, nvr)) elif not wrapper: # only enforce the srpm name matching the build for non-wrapper rpms - srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % buildinfo + nvrinfo = buildinfo.copy() + if draft: + # for draft build the release is buildinfo.extra.draft.target_release + target_release = (buildinfo.get('extra') or {}).get('draft', {}).get('target_release') + if not target_release: + raise koji.GenericError( + f'target release of draft build not found in extra of build: {buildinfo}' + ) + nvrinfo['release'] = target_release + srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % nvrinfo # either the sourcerpm field should match the build, or the filename # itself (for the srpm) if rpminfo['sourcepackage'] != 1: @@ -6320,6 +6423,11 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): raise koji.GenericError("srpm mismatch for %s: %s (expected %s)" % (fn, basename, srpmname)) + state = koji.BUILD_STATES[buildinfo['state']] + if state in ('FAILED', 'CANCELED', 'DELETED'): + nvr = "%(name)s-%(version)s-%(release)s" % buildinfo + raise koji.GenericError("Build is %s: %s" % (state, nvr)) + # if we're adding an rpm to it, then this build is of rpm type # harmless if build already has this type new_typed_build(buildinfo, 'rpm') @@ -6336,6 +6444,11 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): if fileinfo is not None: extra = fileinfo.get('extra') if extra is not None: + if draft: + # simply deny draft + raise koji.GenericError( + f'When importing draft rpm, cannot specify extra in fileinfo: {fileinfo}' + ) rpminfo['extra'] = json.dumps(extra) koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo, @@ -6391,6 +6504,7 @@ def cg_init_build(cg, data): other values will be ignored anyway (owner, state, ...) :return: dict with build_id and token """ + reject_draft(data) assert_cg(cg) cg_id = lookup_name('content_generator', cg, strict=True)['id'] data['owner'] = context.session.user_id @@ -6670,6 +6784,10 @@ class CG_Importer(object): if (koji.BUILD_STATES[buildinfo['state']] not in ('CANCELED', 'FAILED')): raise koji.GenericError("Build already exists: %r" % buildinfo) # note: the checks in recycle_build will also apply when we call new_build later + + if buildinfo: + reject_draft(buildinfo) + # gather needed data buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra', 'source']) if 'build_id' in metadata['build']: @@ -6896,6 +7014,8 @@ class CG_Importer(object): if 'id' in comp: # not in metadata spec, and will confuse get_rpm raise koji.GenericError("Unexpected 'id' field in component") + # rpm is no more unique with NVRA as draft build is introduced + # TODO: we should consider how to handle them once draft build is enabled for CG rinfo = get_rpm(comp, strict=False) if not rinfo: # XXX - this is a temporary workaround until we can better track external refs @@ -7286,6 +7406,7 @@ def merge_scratch(task_id): if not build: raise koji.ImportError('no such build: %(name)s-%(version)s-%(release)s' % build_nvr) + reject_draft(build, koji.ImportError(f"build to import is a draft build: {build['nvr']}")) if build['state'] != koji.BUILD_STATES['COMPLETE']: raise koji.ImportError('%s did not complete successfully' % build['nvr']) if not build['task_id']: @@ -7557,6 +7678,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No buildroot_id: the id of the buildroot the archive was built in (may be None) fileinfo: content generator metadata for file (may be None) """ + reject_draft(buildinfo) if fileinfo is None: fileinfo = {} @@ -8370,7 +8492,8 @@ def query_history(tables=None, **kwargs): return ret -def untagged_builds(name=None, queryOpts=None): +@convert_draft_option +def untagged_builds(name=None, queryOpts=None, *, draft=DRAFT_FLAG.ALL): """Returns the list of untagged builds""" st_complete = koji.BUILD_STATES['COMPLETE'] # following can be achieved with simple query but with @@ -8387,6 +8510,8 @@ def untagged_builds(name=None, queryOpts=None): if name is not None: clauses.append('package.name = %(name)s') + append_draft_clause(draft, clauses) + query = QueryProcessor(tables=['build', 'package'], columns=['build.id', 'package.name', 'build.version', 'build.release'], aliases=['id', 'name', 'version', 'release'], @@ -8671,6 +8796,7 @@ def _delete_build(binfo): update = UpdateProcessor('build', values=values, clauses=['id=%(build_id)i'], data={'state': st_deleted}) update.execute() + _clean_draft_link(binfo) # now clear the build dir builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): @@ -8686,9 +8812,12 @@ def reset_build(build): WARNING: this function is highly destructive. use with care. nulls task_id sets state to CANCELED + sets volume to DEFAULT clears all referenced data in other tables, including buildroot and archive component tables + draft and extra are kept + after reset, only the build table entry is left """ # Only an admin may do this @@ -8767,6 +8896,8 @@ def reset_build(build): update = UpdateProcessor('build', clauses=['id=%(id)s'], values={'id': binfo['id']}, data={'state': binfo['state'], 'task_id': None, 'volume_id': 0}) update.execute() + + _clean_draft_link(binfo) # now clear the build dir builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): @@ -9649,6 +9780,20 @@ class IsBuildOwnerTest(koji.policy.BaseSimpleTest): return False +class IsDraftTest(koji.policy.BaseSimpleTest): + """Check if the build is a draft build""" + name = "is_draft" + + def run(self, data): + if 'draft' in data: + return data['draft'] + if 'build' in data: + build = get_build(data['build']) + return build.get('draft', False) + # default... + return False + + class UserInGroupTest(koji.policy.BaseSimpleTest): """Check if user is in group(s) @@ -10798,23 +10943,24 @@ class RootExports(object): build = get_build(build, strict=True) return apply_volume_policy(build, strict) - def createEmptyBuild(self, name, version, release, epoch, owner=None): + def createEmptyBuild(self, name, version, release, epoch, owner=None, draft=False): """Creates empty build entry :param str name: build name :param str version: build version :param str release: release version :param str epoch: epoch version - :param userInfo: a str (Kerberos principal or name) or an int (user id) + :param owner: a str (Kerberos principal or name) or an int (user id) or a dict: - id: User's ID - name: User's name - krb_principal: Kerberos principal + :param bool draft: create a draft build or not :return: int build ID """ context.session.assertPerm('admin') data = {'name': name, 'version': version, 'release': release, - 'epoch': epoch} + 'epoch': epoch, 'draft': draft} if owner is not None: data['owner'] = owner return new_build(data) @@ -10836,7 +10982,9 @@ class RootExports(object): - artifact_id: Artifact's ID - version: version :raises: GenericError if type for build_info is not dict, when build isn`t existing. + :raises: GenericError if draft: True in buildinfo, when build isn't existing. :raises: GenericError if build info doesn't have mandatory keys. + :raises: GenericError if build is a draft, when it's existing. """ context.session.assertPerm('maven-import') if not context.opts.get('EnableMaven'): @@ -10845,11 +10993,13 @@ class RootExports(object): if not build: if not isinstance(build_info, dict): raise koji.GenericError('Invalid type for build_info: %s' % type(build_info)) + reject_draft(build_info) try: build_id = new_build(dslice(build_info, ('name', 'version', 'release', 'epoch'))) except KeyError as cm: raise koji.GenericError("Build info doesn't have mandatory %s key" % cm) build = get_build(build_id, strict=True) + reject_draft(build) new_maven_build(build, maven_info) def createWinBuild(self, build_info, win_info): @@ -10866,7 +11016,9 @@ class RootExports(object): :param dict win_info: - platform: build platform :raises: GenericError if type for build_info is not dict, when build isn`t existing. + :raises: GenericError if draft: True in buildinfo, when build isn't existing. :raises: GenericError if build info doesn't have mandatory keys. + :raises: GenericError if build is a draft, when it's existing. """ context.session.assertPerm('win-import') if not context.opts.get('EnableWin'): @@ -10875,11 +11027,13 @@ class RootExports(object): if not build: if not isinstance(build_info, dict): raise koji.GenericError('Invalid type for build_info: %s' % type(build_info)) + reject_draft(build_info) try: build_id = new_build(dslice(build_info, ('name', 'version', 'release', 'epoch'))) except KeyError as cm: raise koji.GenericError("Build info doesn't have mandatory %s key" % cm) build = get_build(build_id, strict=True) + reject_draft(build) new_win_build(build, win_info) def createImageBuild(self, build_info): @@ -10895,35 +11049,47 @@ class RootExports(object): - release: build release - epoch: build epoch :raises: GenericError if type for build_info is not dict, when build isn`t existing. + :raises: GenericError if draft: True in buildinfo, when build isn't existing. :raises: GenericError if build info doesn't have mandatory keys. + :raises: GenericError if build is a draft, when it's existing. """ context.session.assertPerm('image-import') build = get_build(build_info) if not build: if not isinstance(build_info, dict): raise koji.GenericError('Invalid type for build_info: %s' % type(build_info)) + reject_draft(build_info) try: build_id = new_build(dslice(build_info, ('name', 'version', 'release', 'epoch'))) except KeyError as cm: raise koji.GenericError("Build info doesn't have mandatory %s key" % cm) build = get_build(build_id, strict=True) + reject_draft(build) new_image_build(build) - def importRPM(self, path, basename): + def importRPM(self, path, basename, build=None, draft=False): """Import an RPM into the database. The file must be uploaded first. """ context.session.assertPerm('admin') + if build: + # can get unique nvr from rpm header when importing to regular build + if not draft: + raise koji.ParameterError( + "Only support specifying build when importing a draft rpm" + ) + build = get_build(build, strict=True) uploadpath = koji.pathinfo.work() fn = "%s/%s/%s" % (uploadpath, path, basename) if not os.path.exists(fn): raise koji.GenericError("No such file: %s" % fn) - rpminfo = import_rpm(fn) + rpminfo = import_rpm(fn, buildinfo=build, draft=draft) import_rpm_file(fn, rpminfo['build'], rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) for tag in list_tags(build=rpminfo['build_id']): set_tag_update(tag['id'], 'IMPORT') + return rpminfo def mergeScratch(self, task_id): """Import the rpms generated by a scratch build, and associate @@ -11709,38 +11875,46 @@ class RootExports(object): raise koji.GenericError("Finished task's priority can't be updated") task.setPriority(priority, recurse=recurse) + @convert_draft_option def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, - owner=None, type=None, strict=True, extra=False): + owner=None, type=None, strict=True, extra=False, *, draft=DRAFT_FLAG.ALL): """List builds tagged with tag. :param int|str tag: tag name or ID number :param int event: event ID :param bool inherit: If inherit is True, follow the tag hierarchy and return - a list of tagged builds for all tags in the tree + a list of tagged builds for all tags in the tree :param str prefix: only builds whose package name starts with that prefix - :param bool|int latest: True for latest build per package, - N to get N latest builds per package. + :param bool|int latest: True for latest build per package, + N to get N latest builds per package. :param str package: only builds of the specified package :param owner: only builds of the specified owner :param str type: only builds of the given btype (such as maven or image) :param bool strict: If tag doesn't exist, an exception is raised, - unless strict is False in which case returns an empty list. + unless strict is False in which case returns an empty list. :param bool extra: Set to "True" to get the build extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds """ # lookup tag id tag = get_tag(tag, strict=strict, event=event) if not tag: return [] results = readTaggedBuilds(tag['id'], event, inherit=inherit, latest=latest, - package=package, owner=owner, type=type, extra=extra) + package=package, owner=owner, type=type, extra=extra, + draft=draft) if prefix: prefix = prefix.lower() results = [build for build in results if build['package_name'].lower().startswith(prefix)] return results + @convert_draft_option def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, - rpmsigs=False, owner=None, type=None, strict=True, extra=True): + rpmsigs=False, owner=None, type=None, strict=True, extra=True, + *, draft=DRAFT_FLAG.ALL): """List rpms and builds within tag. :param int|str tag: tag name or ID number @@ -11759,6 +11933,10 @@ class RootExports(object): :param bool strict: If tag doesn't exist, an exception is raised, unless strict is False in which case returns an empty list. :param bool extra: Set to "False" to skip the rpms extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular """ # lookup tag id tag = get_tag(tag, strict=strict, event=event) @@ -11766,7 +11944,7 @@ class RootExports(object): return [] return readTaggedRPMS(tag['id'], event=event, inherit=inherit, latest=latest, package=package, arch=arch, rpmsigs=rpmsigs, owner=owner, - type=type, extra=extra) + type=type, extra=extra, draft=draft) def listTaggedArchives(self, tag, event=None, inherit=False, latest=False, package=None, type=None, strict=True, extra=True): @@ -11791,10 +11969,11 @@ class RootExports(object): return readTaggedArchives(tag['id'], event=event, inherit=inherit, latest=latest, package=package, type=type, extra=extra) + @convert_draft_option def listBuilds(self, packageID=None, userID=None, taskID=None, prefix=None, state=None, volumeID=None, source=None, createdBefore=None, createdAfter=None, completeBefore=None, completeAfter=None, type=None, typeInfo=None, - queryOpts=None, pattern=None, cgID=None): + queryOpts=None, pattern=None, cgID=None, *, draft=DRAFT_FLAG.ALL): """ Return a list of builds that match the given parameters @@ -11830,6 +12009,10 @@ class RootExports(object): fields are matched For type=win, the provided platform fields are matched + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns: Returns a list of maps. Each map contains the following keys: @@ -11837,6 +12020,7 @@ class RootExports(object): - version - release - epoch + - draft - state - package_id - package_name @@ -11871,7 +12055,9 @@ class RootExports(object): """ fields = [('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), - ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.epoch', 'epoch'), + ('build.draft', 'draft'), + ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.source', 'source'), @@ -11977,6 +12163,7 @@ class RootExports(object): btype_id = btype['id'] joins.append('build_types ON build.id = build_types.build_id ' 'AND btype_id = %(btype_id)s') + append_draft_clause(draft, clauses) query = QueryProcessor(columns=[pair[0] for pair in fields], aliases=[pair[1] for pair in fields], @@ -11986,7 +12173,8 @@ class RootExports(object): return query.iterate() - def getLatestBuilds(self, tag, event=None, package=None, type=None): + @convert_draft_option + def getLatestBuilds(self, tag, event=None, package=None, type=None, *, draft=DRAFT_FLAG.ALL): """List latest builds for tag (inheritance enabled, wrapper of readTaggedBuilds) :param int tag: tag ID @@ -11994,15 +12182,22 @@ class RootExports(object): :param int package: filter on package name :param str type: restrict the list to builds of the given type. Currently the supported types are 'maven', 'win', 'image', or any custom content generator btypes. + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns [dict]: list of buildinfo dicts """ if not isinstance(tag, int): # lookup tag id tag = get_tag_id(tag, strict=True) - return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type) + return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type, + draft=draft) - def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None): + @convert_draft_option + def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None, + *, draft=DRAFT_FLAG.ALL): """List latest RPMS for tag (inheritance enabled, wrapper of readTaggedBuilds) :param int|str tag: The tag name or ID to search @@ -12015,6 +12210,10 @@ class RootExports(object): :param bool rpmsigs: query will return one record per rpm/signature combination :param str type: Filter by build type. Supported types are 'maven', 'win', and 'image'. + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular :returns: a two-element list. The first element is the list of RPMs, and the second element is the list of builds. """ @@ -12023,7 +12222,7 @@ class RootExports(object): # lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, - latest=True, rpmsigs=rpmsigs, type=type) + latest=True, rpmsigs=rpmsigs, type=type, draft=draft) def getLatestMavenArchives(self, tag, event=None, inherit=True): """Return a list of the latest Maven archives in the tag, as of the given event @@ -12149,6 +12348,7 @@ class RootExports(object): - release - arch - epoch + - draft - payloadhash - size - buildtime @@ -13455,6 +13655,118 @@ class RootExports(object): koji.plugin.run_callbacks('postBuildStateChange', attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) + def promoteBuild(self, build, strict=True, force=False): + """Promote a draft build to a regular build. + + - The build type is limited to rpm so far. + - The promoting action cannot be revoked. + - The release wil be changed to the target one, so build_id isn't changed + - buildpath will be changed as well and the old build path will symlink + to the new one, so both paths still will be existing until deleted. + + :param build: A build ID (int), a NVR (string), or a dict containing + "name", "version" and "release" of a draft build + :type build: int, str, dict + :param bool strict: Whether raising Error (default) + or depress it by return None + :param bool force: If False (default), Koji will check this + operation against the draft_promotion hub policy. If hub + policy does not allow the current user to promote the draft build, + then this method will raise an error (strict=True) or return None + (strict=False). + If True, then this method will bypass hub policy settings. + Only admin users can set force to True. + :returns: latest build info, or None if any failure with strict=False + :rtype: dict + """ + context.session.assertLogin() + user = self.getLoggedInUser() + + binfo = get_build(build, strict=strict) + if not binfo: + return None + if not binfo.get('draft'): + if strict: + raise koji.GenericError(f'Not a draft build: {binfo}') + else: + return None + old_release = binfo['release'] + extra = binfo.get('extra') or {} + # get target release in binfo.extra + draft_info = extra.get('draft', {}).copy() + target_release = draft_info.get('target_release') + if not target_release: + if strict: + raise koji.GenericError( + f'draft.target_release not found in extra of build: {binfo}' + ) + else: + return None + target_build = dslice(binfo, ['name', 'version']) + target_build['release'] = target_release + old_build = get_build(target_build) + if old_build: + if strict: + raise koji.GenericError(f'Target build already exists: {old_build}') + else: + return None + # policy checks + policy_data = { + 'build': binfo['id'], + 'target_release': target_release + } + assert_policy('draft_promotion', policy_data, force=force) + # volume check, deny it if volume is changed as it's only allowed for admin + # after building, see applyVolumePolicy + new_volume = apply_volume_policy(target_build, strict=False, dry_run=True) + if new_volume is not None and new_volume['id'] != binfo['volume_id']: + # probably we can just apply the volume change here + if strict: + raise koji.GenericError( + f'Denial as volume will be changed to {new_volume["name"]}' + ) + else: + return None + + koji.plugin.run_callbacks( + 'preBuildPromote', + draft_release=old_release, + target_release=target_release, + build=binfo, + user=user + ) + + # temp solution with python generated time, maybe better to be an event? + now = datetime.datetime.now() + + extra['draft'] = draft_info + draft_info['promoted'] = True + draft_info['old_release'] = old_release + draft_info['promotion_time'] = encode_datetime(now) + draft_info['promotion_ts'] = now.timestamp() + draft_info['promoter'] = user['name'] + extra = json.dumps(extra) + + update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) + update.set(draft=False, release=target_release, extra=extra) + update.execute() + + new_binfo = get_build(binfo['id'], strict=strict) + move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) + ensure_volume_symlink(new_binfo) + + for tag in list_tags(build=binfo['id']): + set_tag_update(tag['id'], 'DRAFT_PROMOTION') + + koji.plugin.run_callbacks( + 'postBuildPromote', + draft_release=old_release, + target_release=target_release, + build=new_binfo, + user=user + ) + return new_binfo + def count(self, methodName, *args, **kw): """Execute the XML-RPC method with the given name and count the results. A method return value of None will return O, a return value of type "list", "tuple", or @@ -14014,6 +14326,7 @@ class BuildRoot(object): ('epoch', 'epoch'), ('arch', 'arch'), ('build_id', 'build_id'), + ('draft', 'draft'), ('external_repo_id', 'external_repo_id'), ('external_repo.name', 'external_repo_name'), ) @@ -14040,7 +14353,7 @@ class BuildRoot(object): data = add_external_rpm(an_rpm, location, strict=False) # will add if missing, compare if not else: - data = get_rpm(an_rpm, strict=True) + data = get_rpm(an_rpm, strict=True, build=an_rpm.get('build', None)) rpm_id = data['id'] if update and rpm_id in current: # ignore duplicate packages for updates @@ -14604,14 +14917,14 @@ class HostExports(object): new_typed_build(binfo, 'rpm') return build_id - def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None): + def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None, draft=False): """Import final build contents into the database""" # sanity checks host = Host() host.verify() task = Task(task_id) task.assertHost(host.id) - result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs) + result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs, draft=False) build_notification(task_id, build_id) return result @@ -15684,3 +15997,64 @@ def create_rpm_checksum(rpm_id, sigkey, chsum_dict): insert.add_record(rpm_id=rpm_id, sigkey=sigkey, checksum=chsum, checksum_type=koji.CHECKSUM_TYPES[func]) insert.execute() + + +def reject_draft(buildinfo, error=None): + """block draft build + + TODO: remove this once draft build is open for all build types + + :param dict buildinfo: buildinfo dict + :param koji.GenericError error: the error raised if not a draft build, + defaults to None to raise the default "unsupported" error + :raises error: default or specified by input error when draft==True in buldinfo + """ + if buildinfo.get('draft'): + if error is None: + error = koji.GenericError("Draft build not supported") + raise error + + +def append_draft_clause(draft, clauses, table=None): + """append proper clause in build/rpm query for draft flags + + DRAFT=1: append "draft IS True" + REGULAR=2: append "draft IS NOT True" + ALL(DRAFT|REGULAR)=3: do nothing + + :param DRAFT_FLAGS draft: draft bit flag(s) + :param list clauses: clauses list to construct query by QueryProcessor, which the draft clause + to append + """ + if not isinstance(draft, (DRAFT_FLAG)): + raise koji.ParameterError(f'draft must be a DRAFT_FLAG, but got {draft}') + if not table: + table = '' + else: + table += '.' + if DRAFT_FLAG.ALL in draft: + return + if DRAFT_FLAG.DRAFT in draft: + clauses.append(f'{table}draft IS TRUE') + if DRAFT_FLAG.REGULAR in draft: + # null is included + clauses.append(f'{table}draft IS NOT TRUE') + + +def _clean_draft_link(promoted_build): + """remove the symlink of old builddir to prmoted builddir""" + draft_info = (promoted_build.get('extra') or {}).get('draft', {}) + if not (draft_info and draft_info.get('promoted')): + # skipped as it's not a promoted build. + return + old_release = draft_info.get('old_release') + if not old_release: + raise koji.GenericError( + f"extra.draft.old_release not found in build: {promoted_build['nvr']}" + ) + draft_buildinfo = dslice(promoted_build, ['name', 'version', 'volume_name']) + draft_buildinfo['release'] = old_release + draft_builddir = koji.pathinfo.build(draft_buildinfo) + # only unlink whe its a symlink. + if os.path.islink(draft_builddir): + os.unlink(draft_builddir) diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py index 64ff8bc..36ec94b 100644 --- a/kojihub/kojixmlrpc.py +++ b/kojihub/kojixmlrpc.py @@ -610,6 +610,11 @@ _default_policies = { 'priority': ''' all :: stay ''', + 'draft_promotion': ''' + has_perm draft-promoter :: allow + is_build_owner :: allow + all :: deny Only draft-promoter and build owner can do this via default policy + ''' } diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index 0489ff4..c28aa4d 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -320,6 +320,22 @@ def prep_repo_done(cbtype, *args, **kws): queue_msg(address, props, kws) +@convert_datetime +@callback('postBuildPromote') +def prep_build_promote(cbtype, *args, **kws): + kws['build'] = _strip_extra(kws['build']) + address = 'build.promote' + props = {'type': cbtype[4:], + 'build_id': kws['build']['id'], + 'name': kws['build']['name'], + 'version': kws['build']['version'], + 'release': kws['build']['release'], + 'draft_release': kws['draft_release'], + 'target_release': kws['target_release'], + 'user': kws['user']['name']} + queue_msg(address, props, kws) + + def _send_msgs(urls, msgs, CONFIG): random.shuffle(urls) for url in urls: diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index ac02893..0250ad8 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -2,6 +2,7 @@ -- from version 1.33 to 1.34 BEGIN; + -- scheduler tables CREATE TABLE scheduler_task_runs ( id SERIAL NOT NULL PRIMARY KEY, @@ -48,4 +49,27 @@ BEGIN; ) WITHOUT OIDS; INSERT INTO locks(name) VALUES('scheduler'); + + -- draft builds + INSERT INTO permissions (name, description) VALUES ('draft-promoter', 'The permission required in the default "draft_promotion" hub policy rule to promote draft build.'); + + ALTER TABLE build ADD COLUMN draft BOOLEAN NOT NULL DEFAULT 'false'; + ALTER TABLE build ADD CONSTRAINT draft_for_rpminfo UNIQUE (id, draft); + ALTER TABLE build ADD CONSTRAINT draft_release_sane CHECK + ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) + OR NOT draft); + + ALTER TABLE rpminfo ADD COLUMN draft BOOLEAN; + ALTER TABLE rpminfo DROP CONSTRAINT rpminfo_build_id_fkey; + ALTER TABLE rpminfo ADD CONSTRAINT rpminfo_build_id_draft_fkey + FOREIGN KEY (build_id, draft) REFERENCES build(id, draft) + ON UPDATE CASCADE; + ALTER TABLE rpminfo DROP CONSTRAINT rpminfo_unique_nvra; + ALTER TABLE rpminfo ADD CONSTRAINT build_id_draft_external_repo_id_sane + CHECK ((draft IS NULL AND build_id IS NULL AND external_repo_id <> 0) + OR (draft IS NOT NULL AND build_id IS NOT NULL AND external_repo_id = 0)); + CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft + ON rpminfo(name,version,release,arch,external_repo_id) + WHERE draft IS NOT TRUE; + COMMIT; diff --git a/schemas/schema.sql b/schemas/schema.sql index a007b2c..b03b891 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -66,6 +66,7 @@ INSERT INTO permissions (name, description) VALUES ('tag', 'Manage packages in t INSERT INTO permissions (name, description) VALUES ('target', 'Add, edit, and remove targets.'); INSERT INTO permissions (name, description) VALUES ('win-admin', 'The default hub policy rule for "vm" requires this permission to trigger Windows builds.'); INSERT INTO permissions (name, description) VALUES ('win-import', 'Import win archives.'); +INSERT INTO permissions (name, description) VALUES ('draft-promoter', 'The permission required in the default "draft_promotion" hub policy rule to promote draft build.'); CREATE TABLE user_perms ( user_id INTEGER NOT NULL REFERENCES users(id), @@ -279,11 +280,12 @@ CREATE TABLE content_generator ( -- null, or may point to a deleted task. CREATE TABLE build ( id SERIAL NOT NULL PRIMARY KEY, - volume_id INTEGER NOT NULL REFERENCES volume (id), + volume_id INTEGER NOT NULL REFERENCES volume (id), pkg_id INTEGER NOT NULL REFERENCES package (id) DEFERRABLE, version TEXT NOT NULL, release TEXT NOT NULL, epoch INTEGER, + draft BOOLEAN NOT NULL DEFAULT 'false', source TEXT, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), start_time TIMESTAMPTZ, @@ -294,8 +296,11 @@ CREATE TABLE build ( cg_id INTEGER REFERENCES content_generator(id), extra TEXT, CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), + CONSTRAINT draft_for_rpminfo UNIQUE (id, draft), CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR - (state != 0 AND completion_time IS NOT NULL)) + (state <> 0 AND completion_time IS NOT NULL)), + CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) OR + NOT draft) ) WITHOUT OIDS; CREATE INDEX build_by_pkg_id ON build (pkg_id); @@ -721,22 +726,28 @@ CREATE TABLE group_package_listing ( -- we don't store filename b/c filename should be N-V-R.A.rpm CREATE TABLE rpminfo ( id SERIAL NOT NULL PRIMARY KEY, - build_id INTEGER REFERENCES build (id), + build_id INTEGER, buildroot_id INTEGER REFERENCES buildroot (id), name TEXT NOT NULL, version TEXT NOT NULL, release TEXT NOT NULL, epoch INTEGER, arch VARCHAR(16) NOT NULL, + draft BOOLEAN, external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), payloadhash TEXT NOT NULL, size BIGINT NOT NULL, buildtime BIGINT NOT NULL, metadata_only BOOLEAN NOT NULL DEFAULT FALSE, extra TEXT, - CONSTRAINT rpminfo_unique_nvra UNIQUE (name,version,release,arch,external_repo_id) + FOREIGN KEY (build_id, draft) REFERENCES build (id, draft) ON UPDATE CASCADE, + CONSTRAINT build_id_draft_external_repo_id_sane CHECK ( + (draft IS NULL AND build_id IS NULL AND external_repo_id <> 0) + OR (draft IS NOT NULL AND build_id IS NOT NULL AND external_repo_id = 0)) ) WITHOUT OIDS; CREATE INDEX rpminfo_build ON rpminfo(build_id); +CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft ON rpminfo(name,version,release,arch,external_repo_id) + WHERE draft IS NOT TRUE; -- index for default search method for rpms, PG11+ can benefit from new include method DO $$ DECLARE version integer; diff --git a/tests/test_builder/data/calls/build_notif_1/message.txt b/tests/test_builder/data/calls/build_notif_1/message.txt index 1693f6a..e46f489 100644 --- a/tests/test_builder/data/calls/build_notif_1/message.txt +++ b/tests/test_builder/data/calls/build_notif_1/message.txt @@ -5,6 +5,7 @@ X-Koji-Tag: f23 X-Koji-Package: sisu X-Koji-Builder: user X-Koji-Status: complete +X-Koji-Draft: False Package: sisu-0.3.0-0.2.M1.fc23 Tag: f23 diff --git a/tests/test_builder/data/calls/build_notif_1/params.json b/tests/test_builder/data/calls/build_notif_1/params.json index 64281ee..12b2ebe 100644 --- a/tests/test_builder/data/calls/build_notif_1/params.json +++ b/tests/test_builder/data/calls/build_notif_1/params.json @@ -22,7 +22,8 @@ "completion_ts": 1424271457.10787, "id": 612609, "volume_name": "DEFAULT", - "nvr": "sisu-0.3.0-0.2.M1.fc23" + "nvr": "sisu-0.3.0-0.2.M1.fc23", + "draft": false }, "target": { "dest_tag": 292, diff --git a/tests/test_cli/test_build.py b/tests/test_cli/test_build.py index fe53995..b66d391 100644 --- a/tests/test_cli/test_build.py +++ b/tests/test_cli/test_build.py @@ -176,6 +176,7 @@ Options: Provide a JSON string of custom metadata to be deserialized and stored under the build's extra.custom_user_metadata field + --draft Build draft build instead """ % (self.progname, self.progname)) # Finally, assert that things were called as we expected. diff --git a/tests/test_cli/test_buildinfo.py b/tests/test_cli/test_buildinfo.py index e33b7c2..c0180f6 100644 --- a/tests/test_cli/test_buildinfo.py +++ b/tests/test_cli/test_buildinfo.py @@ -80,6 +80,37 @@ Tags: self.session.listRPMs.assert_called_once_with(buildID=self.buildinfo['id']) self.assertEqual(self.session.listArchives.call_count, 4) + @mock.patch('sys.stdout', new_callable=StringIO) + def test_buildinfo_draft(self, stdout): + build = 'test-build-1-1' + binfo = copy.deepcopy(self.buildinfo) + binfo['draft'] = True + self.session.getBuild.return_value = binfo + self.session.getTaskInfo.return_value = self.taskinfo + self.session.listTags.return_value = [] + self.session.getMavenBuild.return_value = None + self.session.getWinBuild.return_value = None + self.session.listArchives.return_value = [] + self.session.listRPMs.return_value = [] + expected_stdout = """BUILD: test-build-1-1 [1] +Draft: YES +State: COMPLETE +Built by: kojiadmin +Volume: DEFAULT +Task: 8 build (target, src) +Finished: Thu, 04 Mar 2021 14:45:40 UTC +Tags: +""" + anon_handle_buildinfo(self.options, self.session, [build]) + self.assert_console_message(stdout, expected_stdout) + self.session.listTags.assert_called_once_with(build) + self.session.getBuild.assert_called_once_with(build) + self.session.getTaskInfo.assert_called_once_with(self.buildinfo['task_id'], request=True) + self.session.getMavenBuild.assert_called_once_with(self.buildinfo['id']) + self.session.getWinBuild.assert_called_once_with(self.buildinfo['id']) + self.session.listRPMs.assert_called_once_with(buildID=self.buildinfo['id']) + self.assertEqual(self.session.listArchives.call_count, 4) + def test_buildinfo_more_build_with_non_exist_build(self): build = 'test-build-1-1' non_exist_build = 'test-build-11-12' diff --git a/tests/test_cli/test_import.py b/tests/test_cli/test_import.py index e91a5a7..5fbb5f9 100644 --- a/tests/test_cli/test_import.py +++ b/tests/test_cli/test_import.py @@ -87,15 +87,22 @@ class TestImport(utils.CliTestCase): def __do_import_test(self, options, session, arguments, **kwargs): expected = kwargs.get('expected', None) rpm_header = kwargs.get('rpm_header', {}) + rpm_headers = kwargs.get('rpm_headers', []) + if not rpm_headers: + rpm_headers = [rpm_header] fake_srv_path = kwargs.get('srv_path', '/path/to/server/import') upload_rpm_mock = kwargs.get('upload_rpm_mock', session.uploadWrapper) + getrpm_called = kwargs.get('getrpm_called', True) + getrpm_calls = kwargs.get('getrpm_calls', []) + import_opts = kwargs.get('import_opts', {}) + import_rpm_calls = kwargs.get('import_rpm_calls', None) with mock.patch('koji.get_header_fields') as get_header_fields_mock: with mock.patch('koji_cli.commands.unique_path') as unique_path_mock: with mock.patch('koji_cli.commands.activate_session') as activate_session_mock: with mock.patch('sys.stdout', new_callable=six.StringIO) as stdout: with upload_rpm_mock: - get_header_fields_mock.return_value = rpm_header + get_header_fields_mock.side_effect = rpm_headers unique_path_mock.return_value = fake_srv_path handle_import(options, session, arguments) @@ -104,21 +111,33 @@ class TestImport(utils.CliTestCase): # check mock calls activate_session_mock.assert_called_with(session, options) - get_header_fields_mock.assert_called_with( - arguments[0], - ('name', 'version', 'release', 'epoch', - 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') - ) - - session.getRPM.assert_called_with( - dict((k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name']) - ) + + get_header_fields_calls = [ + mock.call(arguments[i], + ('name', 'version', 'release', 'epoch', + 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') + ) for i in range(len(rpm_headers) - 1) + ] + + get_header_fields_mock.assert_has_calls(get_header_fields_calls) + + if getrpm_calls: + session.getRPM.assert_has_calls(getrpm_calls) + elif getrpm_called: + session.getRPM.assert_called_with( + dict((k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name']) + ) + else: + session.getRPM.assert_not_called() unique_path_mock.assert_called_with('cli-import') upload_rpm_mock.assert_called_with(arguments[0], self.fake_srv_dir) - session.importRPM.assert_called_with( - self.fake_srv_dir, os.path.basename(arguments[0])) + if import_rpm_calls: + session.importRPM.assert_has_calls(import_rpm_calls) + else: + session.importRPM.assert_called_with( + self.fake_srv_dir, os.path.basename(arguments[0]), **import_opts) # reset for next test activate_session_mock.reset_mock() @@ -136,6 +155,7 @@ class TestImport(utils.CliTestCase): expected = kwargs.get('expected', None) expected_warn = kwargs.get('expected_warn', None) rpm_header = kwargs.get('rpm_header', {}) + getrpm_called = kwargs.get('getrpm_called', True) with mock.patch('koji.get_header_fields') as get_header_fields_mock: get_header_fields_mock.return_value = rpm_header @@ -152,11 +172,13 @@ class TestImport(utils.CliTestCase): ('name', 'version', 'release', 'epoch', 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') ) - - session.getRPM.assert_called_with( - dict((k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name']) - ) + if getrpm_called: + session.getRPM.assert_called_with( + dict((k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name']) + ) + else: + session.getRPM.assert_not_called() session.uploadWrapper.assert_not_called() session.importRPM.assert_not_called() @@ -677,6 +699,225 @@ class TestImport(utils.CliTestCase): activate_session=None) activate_session_mock.assert_not_called() + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_src_rpm_with_create_draft( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import SRPM import with creating draft build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--create-draft'] + options = mock.MagicMock() + session = mock.MagicMock() + session.importRPM.return_value = {'build': {'nvr': 'a-draft-build', 'draft': True}} + + nvr = '%(name)s-%(version)s-%(release)s' % self.srpm_header + + # Case 1. import src rpm with --create-draft + # result: success + expected = "Will create draft build instead if desired nvr doesn't exist\n" + expected += "Will create draft build with target nvr: %s while importing\n" % nvr + expected += "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + expected += "Draft build: a-draft-build created\n" + + self.__do_import_test( + options, session, arguments, + rpm_header=self.srpm_header, + expected=expected, + getrpm_called=False, + import_opts={'draft': True}) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_binary_rpm_with_create_draft( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import binary RPM import with creating draft build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--create-draft'] + options = mock.MagicMock() + session = mock.MagicMock() + + nvr = '%(name)s-%(version)s-%(release)s' % self.rpm_header + + # Case 1. import bin rpm with --create-draft + # result: Aborting import + expected = "Will create draft build instead if desired nvr doesn't exist\n" + expected += "Missing srpm for draft build creating with target nvr: %s\n" % nvr + expected += "Aborting import\n" + self.__skip_import_test( + options, session, arguments, + rpm_header=self.rpm_header, + expected=expected, + getrpm_called=False) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_src_bin_rpm_with_create_draft( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import SRPM & RPM import with creating draft build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '/path/to/bash-4.4.12-5.fc26.src.rpm', + '--create-draft'] + options = mock.MagicMock() + session = mock.MagicMock() + session.getRPM.return_value = None + session.importRPM.return_value = {'build': {'nvr': 'a-draft-build', 'draft': True}} + + nvr = '%(name)s-%(version)s-%(release)s' % self.srpm_header + + # Case 1. import src & bin rpm with --create-draft + # result: success + expected = "Will create draft build instead if desired nvr doesn't exist\n" + expected += "Will create draft build with target nvr: %s while importing\n" % nvr + expected += "uploading %s... done\n" % arguments[1] + expected += "importing %s... done\n" % arguments[1] + expected += "Draft build: a-draft-build created\n" + expected += "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + + self.__do_import_test( + options, session, arguments, + rpm_headers=[self.rpm_header, self.srpm_header], + expected=expected, + getrpm_calls=[ + mock.call( + {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'x86_64'}, + build=session.importRPM.return_value['build'] + ) + ], + import_rpm_calls=[ + mock.call( + self.fake_srv_dir, os.path.basename(arguments[1]), draft=True + ), + mock.call( + self.fake_srv_dir, os.path.basename(arguments[0]), + build=session.importRPM.return_value['build'], draft=True + ) + ]) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_src_rpm_with_specified_draft_build( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import SRPM import with --draft-build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', 'a-draft-build'] + options = mock.MagicMock() + session = mock.MagicMock() + build = { + 'name': 'bash', + 'version': '4.4.12', + 'release': '5.fc26#draft_123', + 'nvr': 'bash-4.4.12-5.fc26', + 'draft': True, + 'extra': { + 'draft': { + 'target_release': '5.fc26' + } + }, + 'state': self.bstate['COMPLETE'] + } + session.getBuild.return_value = build + + # Case 1. import src rpm with --draft-build + # result: success + expected = "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + + self.__do_import_test( + options, session, arguments, + rpm_header=self.srpm_header, + expected=expected, + getrpm_calls=[mock.call( + {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'src'}, + build=build)], + import_opts={'build': build, 'draft': True}) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_binary_rpm_with_specified_draft_build( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import RPM import with --draft-build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', 'a-draft-build'] + options = mock.MagicMock() + session = mock.MagicMock() + session.getRPM.return_value = None + build = { + 'name': 'bash', + 'version': '4.4.12', + 'release': '5.fc26#draft_123', + 'nvr': 'bash-4.4.12-5.fc26', + 'draft': True, + 'extra': { + 'draft': { + 'target_release': '5.fc26' + } + }, + 'state': self.bstate['COMPLETE'] + } + session.getBuild.return_value = build + + # Case 1. import bin rpm with --draft-build + # result: success + expected = "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + + self.__do_import_test( + options, session, arguments, + rpm_header=self.rpm_header, + expected=expected, + getrpm_calls=[mock.call( + {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'x86_64'}, + build=build)], + import_opts={'build': build, 'draft': True}) + + def test_handle_import_specified_draft_build_invalid(self): + """Test handle_import RPM import with --draft-build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', '286'] + options = mock.MagicMock() + session = mock.MagicMock() + + cases = [ + # build, stderr + (None, "No such build: 286"), + ({'draft': False, 'nvr': 'a-bad-draft'}, "a-bad-draft is not a draft build"), + ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['DELETED']}, + "draft build a-bad-draft is expected as COMPLETE, got DELETED"), + ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], + 'extra': {'draft': {'no_target_release': 'omg'}}}, + "Invalid draft build: a-bad-draft, no draft.target_release found in extra") + ] + for build, expected in cases: + session.getBuild.return_value = build + # result: error + expected += "\n" + self.assert_system_exit( + handle_import, + options, + session, + arguments, + stderr=expected, + activate_session=None, + exit_code=1) + options.reset_mock() + session.reset_mock() + def test_handle_import_help(self): """Test handle_import function help message""" self.assert_help( @@ -691,6 +932,8 @@ Options: --create-build Auto-create builds as needed --src-epoch=SRC_EPOCH When auto-creating builds, use this epoch + --create-draft Auto-create draft builds instead as needed + --draft-build=NVR|ID The target draft build to import to """ % self.progname) diff --git a/tests/test_cli/test_list_builds.py b/tests/test_cli/test_list_builds.py index 0bc8035..4f0875a 100644 --- a/tests/test_cli/test_list_builds.py +++ b/tests/test_cli/test_list_builds.py @@ -609,6 +609,8 @@ Options: pattern) --owner=OWNER List builds built by this owner --volume=VOLUME List builds by volume ID + --draft-only Only list draft builds + --no-draft Only list regular builds -k FIELD, --sort-key=FIELD Sort the list by the named field. Allowed sort keys: build_id, owner_name, state diff --git a/tests/test_cli/test_list_tagged.py b/tests/test_cli/test_list_tagged.py index b30dd5c..3bccfe1 100644 --- a/tests/test_cli/test_list_tagged.py +++ b/tests/test_cli/test_list_tagged.py @@ -42,6 +42,15 @@ class TestCliListTagged(utils.CliTestCase): 'release': '1.el6', 'arch': 'x86_64', 'sigkey': 'sigkey', + 'extra': None}, + {'id': 102, + 'build_id': 2, + 'name': 'rpmA', + 'version': '0.0.1', + 'release': '2.el6', + 'arch': 'x86_64', + 'sigkey': 'sigkey', + 'draft': True, 'extra': None} ], [{'id': 1, 'name': 'packagename', @@ -50,6 +59,15 @@ class TestCliListTagged(utils.CliTestCase): 'nvr': 'n-v-r', 'tag_name': 'tag', 'owner_name': 'owner', + 'extra': 'extra-value-2'}, + {'id': 2, + 'name': 'packagename', + 'version': 'version', + 'release': '2.el6#draft_2', + 'nvr': 'n-v-r', + 'draft': True, + 'tag_name': 'tag', + 'owner_name': 'owner', 'extra': 'extra-value-2'}]] self.session.listTagged.return_value = [{'id': 1, 'name': 'packagename', @@ -77,13 +95,15 @@ Build Tag Built by ---------------------------------------- -------------------- ---------------- n-v-r tag owner """ - args = [self.tag, self.pkg, '--latest', '--inherit', '--event', str(self.event_id)] + args = [self.tag, self.pkg, '--no-draft', '--latest', '--inherit', + '--event', str(self.event_id)] anon_handle_list_tagged(self.options, self.session, args) self.ensure_connection_mock.assert_called_once_with(self.session, self.options) self.session.getTag.assert_called_once_with(self.tag, event=self.event_id) self.session.listTagged.assert_called_once_with( - self.tag, event=self.event_id, inherit=True, latest=True, package=self.pkg) + self.tag, event=self.event_id, inherit=True, latest=True, package=self.pkg, + draft=2) self.session.listTaggedRPMS.assert_not_called() self.assert_console_message(stdout, expected) @@ -94,14 +114,14 @@ n-v-r tag owner ---------------------------------------- -------------------- ---------------- /mnt/koji/packages/packagename/version/1.el6 tag owner """ - args = [self.tag, self.pkg, '--latest', '--inherit', '--paths'] + args = [self.tag, self.pkg, '--latest', '--inherit', '--paths', '--draft-only'] anon_handle_list_tagged(self.options, self.session, args) self.assert_console_message(stdout, expected) self.ensure_connection_mock.assert_called_once_with(self.session, self.options) self.session.getTag.assert_called_once_with(self.tag, event=None) self.session.listTagged.assert_called_once_with( - self.tag, inherit=True, latest=True, package=self.pkg) + self.tag, inherit=True, latest=True, package=self.pkg, draft=1) self.session.listTaggedRPMS.assert_not_called() @mock.patch('sys.stdout', new_callable=six.StringIO) @@ -109,6 +129,7 @@ n-v-r tag owner def test_list_tagged_rpms(self, event_from_opts_mock, stdout): expected = """sigkey rpmA-0.0.1-1.el6.noarch sigkey rpmA-0.0.1-1.el6.x86_64 +sigkey rpmA-0.0.1-2.el6.x86_64 (#draft_2) """ args = [self.tag, self.pkg, '--latest-n=3', '--rpms', '--sigs', '--arch=x86_64', '--arch=noarch'] @@ -129,6 +150,7 @@ sigkey rpmA-0.0.1-1.el6.x86_64 def test_list_tagged_rpms_paths(self, event_from_opts_mock, stdout, os_path_exists, isdir): expected = """/mnt/koji/packages/packagename/version/1.el6/noarch/rpmA-0.0.1-1.el6.noarch.rpm /mnt/koji/packages/packagename/version/1.el6/x86_64/rpmA-0.0.1-1.el6.x86_64.rpm +/mnt/koji/packages/packagename/version/2.el6#draft_2/x86_64/rpmA-0.0.1-2.el6.x86_64.rpm """ args = [self.tag, self.pkg, '--latest-n=3', '--rpms', '--arch=x86_64', '--paths'] @@ -233,6 +255,15 @@ n-v-r tag group self.session.listTaggedRPMS.assert_not_called() self.session.listTagged.assert_not_called() + def test_list_tagged_draft_opts_conflict(self): + self.assert_system_exit( + anon_handle_list_tagged, + self.options, self.session, ['--draft-only', '--no-draft', 'tag', 'pkg1'], + stderr=self.format_error_message("--draft-only conflicts with --no-draft"), + activate_session=None, + exit_code=2) + self.ensure_connection_mock.assert_not_called() + def test_list_tagged_tag_not_found(self): self.session.getTag.return_value = None self.assert_system_exit( @@ -267,4 +298,6 @@ Options: --event=EVENT# query at event --ts=TIMESTAMP query at last event before timestamp --repo=REPO# query at event for a repo + --draft-only Only list draft builds/rpms + --no-draft Only list regular builds/rpms """ % self.progname) diff --git a/tests/test_cli/test_rpminfo.py b/tests/test_cli/test_rpminfo.py index 3e2beb3..8e7aebb 100644 --- a/tests/test_cli/test_rpminfo.py +++ b/tests/test_cli/test_rpminfo.py @@ -53,6 +53,17 @@ class TestRpminfo(utils.CliTestCase): 'version': '1.1', 'payloadhash': 'b2b95550390e5f213fc25f33822425f7', 'size': 7030} + self.listrpminfos = [{'arch': 'src', + 'build_id': 1, + 'buildroot_id': 3, + 'buildtime': 1615877809, + 'epoch': 7, + 'id': 290, + 'name': 'test-rpm', + 'release': '11', + 'version': '1.1', + 'payloadhash': 'b2b95550390e5f213fc25f33822425f7', + 'size': 7030}] self.error_format = """Usage: %s rpminfo [options] [ ...] (Specify the --help global option for a list of other help options) @@ -74,9 +85,11 @@ class TestRpminfo(utils.CliTestCase): self.session.listBuildroots.return_value = [self.buildroot_info] self.session.getBuild.return_value = self.buildinfo self.session.getRPM.return_value = self.getrpminfo + self.session.listRPMs.return_value = self.listrpminfos expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] +Build: test-rpm-1.1-11 [1] RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm -SRPM: 7:test-rpm-1.1-11 [1] +SRPM: 7:test-rpm-1.1-11 [290] SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm Built: Tue, 16 Mar 2021 06:56:49 UTC SIGMD5: b2b95550390e5f213fc25f33822425f7 @@ -98,6 +111,8 @@ Used in 1 buildroots: rpmID=self.getrpminfo['id']) self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) self.session.getRPM.assert_called_once_with(rpm_nvra) + self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], + arches='src') def test_handle_rpminfo_non_exist_nvra(self): rpm_nvra = 'test-rpm-nvra.arch' @@ -119,9 +134,11 @@ Used in 1 buildroots: self.session.listBuildroots.return_value = [self.buildroot_info] self.session.getBuild.return_value = self.buildinfo self.session.getRPM.side_effect = [None, self.getrpminfo] + self.session.listRPMs.return_value = self.listrpminfos expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] +Build: test-rpm-1.1-11 [1] RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm -SRPM: 7:test-rpm-1.1-11 [1] +SRPM: 7:test-rpm-1.1-11 [290] SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm Built: Tue, 16 Mar 2021 06:56:49 UTC SIGMD5: b2b95550390e5f213fc25f33822425f7 @@ -150,6 +167,45 @@ Used in 1 buildroots: rpmID=self.getrpminfo['id']) self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) self.assertEqual(self.session.getRPM.call_count, 2) + self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], + arches='src') + + + @mock.patch('sys.stdout', new_callable=StringIO) + def test_handle_rpminfo_with_build(self, stdout): + rpm_nvra = 'test-rpm-1.1-11.noarch' + self.session.getBuildroot.return_value = self.buildroot_info + self.session.listBuildroots.return_value = [self.buildroot_info] + self.session.getBuild.return_value = self.buildinfo + self.session.getRPM.return_value = self.getrpminfo + self.session.listRPMs.return_value = self.listrpminfos + expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] +Build: test-rpm-1.1-11 [1] +RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm +SRPM: 7:test-rpm-1.1-11 [290] +SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm +Built: Tue, 16 Mar 2021 06:56:49 UTC +SIGMD5: b2b95550390e5f213fc25f33822425f7 +Size: 7030 +Build ID: 1 +Buildroot: 3 (tag test-tag, arch x86_64, repo 2) +Build Host: kojibuilder +Build Task: 10 +Used in 1 buildroots: + id build tag arch build host + -------- ---------------------------- -------- ----------------------------- + 3 test-tag x86_64 kojibuilder +""" + + anon_handle_rpminfo(self.options, self.session, ['--buildroot', '--build', 'any', rpm_nvra]) + self.assert_console_message(stdout, expected_output) + self.session.getBuildroot.assert_called_once_with(self.getrpminfo['buildroot_id']) + self.session.listBuildroots.assert_called_once_with(queryOpts={'order': 'buildroot.id'}, + rpmID=self.getrpminfo['id']) + self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) + self.session.getRPM.assert_called_once_with(rpm_nvra, build='any') + self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], + arches='src') def test_rpminfo_without_option(self): arguments = [] @@ -171,6 +227,7 @@ Used in 1 buildroots: (Specify the --help global option for a list of other help options) Options: - -h, --help show this help message and exit - --buildroots show buildroots the rpm was used in + -h, --help show this help message and exit + --buildroots show buildroots the rpm was used in + --build=NVR|ID show the rpm(s) in the build """ % self.progname) diff --git a/tests/test_cli/test_wrapper_rpm.py b/tests/test_cli/test_wrapper_rpm.py index bb40cb2..94fbd88 100644 --- a/tests/test_cli/test_wrapper_rpm.py +++ b/tests/test_cli/test_wrapper_rpm.py @@ -237,6 +237,7 @@ Options: --wait Wait on build, even if running in the background --nowait Don't wait on build --background Run the build at a lower priority + --draft Build draft build instead """ % self.progname) diff --git a/tests/test_hub/test_getRPM.py b/tests/test_hub/test_getRPM.py index 984c4e0..1e19570 100644 --- a/tests/test_hub/test_getRPM.py +++ b/tests/test_hub/test_getRPM.py @@ -16,6 +16,10 @@ class TestGetRPM(DBQueryTestCase): self.exports = kojihub.RootExports() self.context = mock.patch('kojihub.kojihub.context').start() self.get_external_repo_id = mock.patch('kojihub.kojihub.get_external_repo_id').start() + self.find_build_id = mock.patch('kojihub.kojihub.find_build_id').start() + + def tearDown(self): + mock.patch.stopall() def test_wrong_type_rpminfo(self): rpminfo = ['test-user'] @@ -31,11 +35,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ['external_repo_id = 0', "rpminfo.id=%(id)s"]) self.assertEqual(query.joins, @@ -50,11 +53,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"]) self.assertEqual(query.joins, @@ -70,11 +72,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"]) self.assertEqual(query.joins, @@ -87,11 +88,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["rpminfo.name=%(name)s AND version=%(version)s " "AND release=%(release)s AND arch=%(arch)s"]) @@ -110,17 +110,36 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["external_repo_id = %(external_repo_id)i", "rpminfo.id=%(id)s"]) self.assertEqual(query.joins, ['external_repo ON rpminfo.external_repo_id = external_repo.id']) self.assertEqual(query.values, rpminfo_data) + + def test_rpm_info_with_build(self): + rpminfo = {'id': 123, 'name': 'testrpm-1.23-4.x86_64.rpm', 'build_id': 101} + self.find_build_id.return_value = 101 + rpminfo_data = rpminfo.copy() + + kojihub.get_rpm(rpminfo, multi=True, build='any') + + self.assertEqual(len(self.queries), 1) + query = self.queries[0] + self.assertEqual(query.tables, ['rpminfo']) + columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] + self.assertEqual(set(query.columns), set(columns)) + self.assertEqual(query.clauses, + ["rpminfo.build_id = %(build_id)s", "rpminfo.id=%(id)s"]) + self.assertEqual(query.joins, + ['external_repo ON rpminfo.external_repo_id = external_repo.id']) + self.assertEqual(query.values, rpminfo_data) class TestGetRPMHeaders(unittest.TestCase): diff --git a/tests/test_hub/test_get_build.py b/tests/test_hub/test_get_build.py index 78c4ef5..ac4fe1e 100644 --- a/tests/test_hub/test_get_build.py +++ b/tests/test_hub/test_get_build.py @@ -39,8 +39,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', @@ -64,8 +64,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', @@ -111,8 +111,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', @@ -138,8 +138,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', diff --git a/tests/test_hub/test_get_next_release.py b/tests/test_hub/test_get_next_release.py index 1ca0962..457e179 100644 --- a/tests/test_hub/test_get_next_release.py +++ b/tests/test_hub/test_get_next_release.py @@ -22,7 +22,8 @@ class TestGetNextRelease(DBQueryTestCase): self.assertEqual(query.tables, ['build']) self.assertEqual(query.joins, ['package ON build.pkg_id = package.id']) self.assertEqual(query.clauses, - ['name = %(name)s', 'state in %(states)s', 'version = %(version)s']) + ['NOT draft', 'name = %(name)s', 'state in %(states)s', + 'version = %(version)s']) self.assertEqual(query.values, {'name': self.binfo['name'], 'version': self.binfo['version'], 'states': (1, 2, 0) diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index 564075b..a81d18f 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -92,6 +92,7 @@ class TestImportBuild(unittest.TestCase): fields = [ 'completion_time', + 'draft', 'epoch', 'extra', 'id', @@ -123,6 +124,7 @@ class TestImportBuild(unittest.TestCase): 'release': 'release', 'pkg_id': mock.ANY, 'id': mock.ANY, + 'draft': False } self._dml.assert_called_once_with(statement, values) diff --git a/tests/test_hub/test_import_rpm.py b/tests/test_hub/test_import_rpm.py index 014bdbb..25630e0 100644 --- a/tests/test_hub/test_import_rpm.py +++ b/tests/test_hub/test_import_rpm.py @@ -42,11 +42,12 @@ class TestImportRPM(unittest.TestCase): 1003: 'epoch', 1006: 'buildtime', 1022: 'arch', - 1044: 'name-version-release.arch', + 1044: 'name-version-release.src.rpm', 1106: 'sourcepackage', 261: 'payload hash', } self.get_build = mock.patch('kojihub.kojihub.get_build').start() + self.new_build = mock.patch('kojihub.kojihub.new_build').start() self.get_rpm_header = mock.patch('koji.get_rpm_header').start() self.new_typed_build = mock.patch('kojihub.kojihub.new_typed_build').start() self.nextval = mock.patch('kojihub.kojihub.nextval').start() @@ -65,6 +66,7 @@ class TestImportRPM(unittest.TestCase): kojihub.import_rpm("this does not exist") def test_import_rpm_failed_build(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' self.get_rpm_header.return_value = self.rpm_header_retval self.get_build.return_value = { 'state': koji.BUILD_STATES['FAILED'], @@ -72,9 +74,11 @@ class TestImportRPM(unittest.TestCase): 'version': 'version', 'release': 'release', } - with self.assertRaises(koji.GenericError): + with self.assertRaises(koji.GenericError) as cm: kojihub.import_rpm(self.filename) + self.assertEqual("Build is FAILED: name-version-release", str(cm.exception)) self.assertEqual(len(self.inserts), 0) + def test_import_rpm_completed_build(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' @@ -94,6 +98,7 @@ class TestImportRPM(unittest.TestCase): 'name': 'name', 'arch': 'arch', 'buildtime': 'buildtime', + 'draft': False, 'payloadhash': '7061796c6f61642068617368', 'epoch': 'epoch', 'version': 'version', @@ -114,7 +119,7 @@ class TestImportRPM(unittest.TestCase): retval = copy.copy(self.rpm_header_retval) retval.update({ 'filename': 'name-version-release.arch.rpm', - 1044: 'name-version-release.src', + 1044: 'name-version-release.src.rpm.bad', 1022: 'src', 1106: 1, }) @@ -133,6 +138,7 @@ class TestImportRPM(unittest.TestCase): 'name': 'name', 'arch': 'src', 'buildtime': 'buildtime', + 'draft': False, 'payloadhash': '7061796c6f61642068617368', 'epoch': 'epoch', 'version': 'version', @@ -149,10 +155,9 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(insert.rawdata, {}) def test_non_exist_file(self): - basename = 'rpm-1-34' self.os_path_exists.return_value = False with self.assertRaises(koji.GenericError) as cm: - kojihub.import_rpm(self.filename, basename) + kojihub.import_rpm(self.filename) self.assertEqual(f"No such file: {self.filename}", str(cm.exception)) self.assertEqual(len(self.inserts), 0) @@ -172,3 +177,211 @@ class TestImportRPM(unittest.TestCase): kojihub.import_rpm(self.src_filename) self.assertEqual("No such build", str(cm.exception)) self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_completed_build(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + self.get_build.return_value = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + } + self.nextval.return_value = 9876 + kojihub.import_rpm(self.filename) + + data = { + 'build_id': 12345, + 'name': 'name', + 'arch': 'arch', + 'buildtime': 'buildtime', + 'draft': False, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + + + def test_import_draft_conflict(self): + with self.assertRaises(koji.GenericError) as cm: + kojihub.import_rpm(self.filename, buildinfo={'id': 1024, 'draft': False}, draft=True) + self.assertEqual("draft property: False of build: 1024 mismatch, True is expected", + str(cm.exception)) + self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_without_buildinfo(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + + with self.assertRaises(koji.GenericError) as cm: + kojihub.import_rpm(self.filename, draft=True) + self.assertEqual(f"Cannot import draft rpm: {self.os_path_basename.return_value}" + " without specifying a build", str(cm.exception)) + self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_non_extra_target_release(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + + buildinfo = { + 'state': koji.BUILD_STATES['DELETED'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + 'draft': True + } + + with self.assertRaises(koji.GenericError) as cm: + kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) + self.assertEqual( + f'target release of draft build not found in extra of build: {buildinfo}', + str(cm.exception) + ) + self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_valid(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + + buildinfo = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + 'draft': True, + 'extra': { + 'draft': { + 'target_release': 'release' + } + } + } + self.nextval.return_value = 9876 + kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) + data = { + 'build_id': 12345, + 'name': 'name', + 'arch': 'arch', + 'buildtime': 'buildtime', + 'draft': True, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + + def test_import_draft_srpm_with_buildinfo(self): + self.os_path_basename.return_value = 'name-version-release.src.rpm' + retval = copy.copy(self.rpm_header_retval) + retval.update({ + 'filename': 'name-version-release.src.rpm', + 1044: 'name-version-release.src.rpm.bad', + 1022: 'src', + 1106: 1, + }) + self.get_rpm_header.return_value = retval + buildinfo = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + 'draft': True, + 'extra': { + 'draft': { + 'target_release': 'release' + } + } + } + self.nextval.return_value = 9876 + kojihub.import_rpm(self.src_filename, buildinfo=buildinfo, draft=True) + data = { + 'build_id': 12345, + 'name': 'name', + 'arch': 'src', + 'buildtime': 'buildtime', + 'draft': True, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + + def test_import_draft_srpm_without_buildinfo(self): + self.os_path_basename.return_value = 'name-version-release.src.rpm' + retval = copy.copy(self.rpm_header_retval) + retval.update({ + 'filename': 'name-version-release.src.rpm', + 1044: 'name-version-release.src.rpm.bad', + 1022: 'src', + 1106: 1, + }) + self.get_rpm_header.return_value = retval + self.get_build.return_value = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 5566, + 'draft': True, + 'extra': { + 'draft': { + 'target_release': 'release' + } + } + } + self.new_build.return_value = 5566 + self.nextval.return_value = 9876 + kojihub.import_rpm(self.src_filename, draft=True) + data = { + 'build_id': 5566, + 'name': 'name', + 'arch': 'src', + 'buildtime': 'buildtime', + 'draft': True, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + self.get_build.assert_called_once_with(5566, strict=True) + self.assertEqual(self.get_build.call_count, 1) \ No newline at end of file diff --git a/tests/test_hub/test_list_builds.py b/tests/test_hub/test_list_builds.py index b54015a..fd80bac 100644 --- a/tests/test_hub/test_list_builds.py +++ b/tests/test_hub/test_list_builds.py @@ -17,6 +17,32 @@ class TestListBuilds(unittest.TestCase): return query def setUp(self): + + # defaults + self.tables= ['build'] + self.columns = [ + 'build.id', 'build.completion_time', + "date_part('epoch', build.completion_time)", + 'events.id', 'events.time', + "date_part('epoch', events.time)", + 'build.draft', + 'build.epoch', + 'build.extra', 'package.name', + "package.name || '-' || build.version || '-' || " + "build.release", 'users.id', 'users.name', 'package.id', + 'package.name', 'build.release', 'build.source', + 'build.start_time', "date_part('epoch', build.start_time)", + 'build.state', 'build.task_id', 'build.version', + 'volume.id', 'volume.name' + ] + self.clauses = ['package.id = %(packageID)i'] + self.joins = [ + 'LEFT JOIN events ON build.create_event = events.id', + 'LEFT JOIN package ON build.pkg_id = package.id', + 'LEFT JOIN volume ON build.volume_id = volume.id', + 'LEFT JOIN users ON build.owner = users.id' + ] + self.maxDiff = None self.exports = kojihub.RootExports() self.query_executeOne = mock.MagicMock() @@ -41,7 +67,8 @@ class TestListBuilds(unittest.TestCase): 'task_id': 879, 'version': '11', 'volume_id': 0, - 'volume_name': 'DEFAULT'}] + 'volume_name': 'DEFAULT', + 'draft': False},] def test_wrong_package(self): package = 'test-package' @@ -58,26 +85,27 @@ class TestListBuilds(unittest.TestCase): self.assertEqual(len(self.queries), 1) args, kwargs = self.QueryProcessor.call_args qp = QP(**kwargs) - self.assertEqual(qp.tables, ['build']) - self.assertEqual(qp.columns, ['build.id', 'build.completion_time', - "date_part('epoch', build.completion_time)", - 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', - 'build.extra', 'package.name', - "package.name || '-' || build.version || '-' || " - "build.release", 'users.id', 'users.name', 'package.id', - 'package.name', 'build.release', 'build.source', - 'build.start_time', "date_part('epoch', build.start_time)", - 'build.state', 'build.task_id', 'build.version', - 'volume.id', 'volume.name']) - self.assertEqual(qp.clauses, ['package.id = %(packageID)i']) - self.assertEqual(qp.joins, ['LEFT JOIN events ON build.create_event = events.id', - 'LEFT JOIN package ON build.pkg_id = package.id', - 'LEFT JOIN volume ON build.volume_id = volume.id', - 'LEFT JOIN users ON build.owner = users.id']) + self.assertEqual(qp.tables, self.tables) + self.assertEqual(qp.columns, self.columns) + self.assertEqual(qp.clauses, self.clauses) + self.assertEqual(qp.joins, self.joins) def test_wrong_user(self): user = 'test-user' self.get_user.return_value = None rv = self.exports.listBuilds(userID=user) self.assertEqual(rv, []) + + def test_draft(self): + package = 'test-package' + package_id = 1 + self.get_package_id.return_value = package_id + self.query_executeOne.return_value = None + self.exports.listBuilds(packageID=package, draft=1) + self.assertEqual(len(self.queries), 1) + args, kwargs = self.QueryProcessor.call_args + qp = QP(**kwargs) + self.assertEqual(qp.tables, self.tables) + self.assertEqual(qp.columns, self.columns) + self.assertEqual(qp.clauses, ['draft IS TRUE'] + self.clauses) + self.assertEqual(qp.joins, self.joins) diff --git a/tests/test_hub/test_new_build.py b/tests/test_hub/test_new_build.py index ab4a270..758da47 100644 --- a/tests/test_hub/test_new_build.py +++ b/tests/test_hub/test_new_build.py @@ -2,6 +2,7 @@ import mock import unittest import koji +from koji.util import dslice import kojihub IP = kojihub.InsertProcessor @@ -23,6 +24,7 @@ class TestNewBuild(unittest.TestCase): self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.recycle_build = mock.patch('kojihub.kojihub.recycle_build').start() self.context = mock.patch('kojihub.kojihub.context').start() + self.find_build_id = mock.patch('kojihub.kojihub.find_build_id').start() def tearDown(self): mock.patch.stopall() @@ -64,6 +66,7 @@ class TestNewBuild(unittest.TestCase): 'start_time': 'NOW', 'state': 1, 'task_id': None, + 'draft': False, 'version': 'test_version', 'volume_id': 0 }) @@ -156,3 +159,48 @@ class TestNewBuild(unittest.TestCase): self.assertEqual(len(self.inserts), 0) self.assertEqual("No such build extra data: %(extra)r" % data, str(cm.exception)) + + def test_draft(self): + data = { + 'owner': 123456, + 'name': 'test_name', + 'version': 'test_version', + 'release': 'test_release', + 'epoch': 'test_epoch', + 'draft': True + } + insert_data = { + 'completion_time': 'NOW', + 'epoch': 'test_epoch', + 'extra': '{"draft": {"target_release": "test_release", "promoted": false}}', + 'id': 108, + 'owner': 123, + 'pkg_id': 54, + 'release': 'test_release#draft_108', + 'source': None, + 'start_time': 'NOW', + 'state': 1, + 'task_id': None, + 'draft': True, + 'version': 'test_version', + 'volume_id': 0 + } + self.nextval.return_value = 108 + self.new_package.return_value = 54 + self.get_user.return_value = {'id': 123} + self.find_build_id.return_value = None + + kojihub.new_build(data) + + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'build') + self.assertEqual(insert.data, insert_data) + self.get_build.assert_called_once_with(108, strict=True) + self.assertEqual(self.get_build.call_count, 1) + self.find_build_id.assert_called_once_with( + { + 'name': 'test_name', + 'version': 'test_version', + 'release': 'test_release#draft_108' + }) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py new file mode 100644 index 0000000..8dc58cf --- /dev/null +++ b/tests/test_hub/test_promote_build.py @@ -0,0 +1,179 @@ +import datetime +import json +import mock +import unittest +import koji +import kojihub + + +UP = kojihub.UpdateProcessor + + +class TestPromoteBuild(unittest.TestCase): + + def getUpdate(self, *args, **kwargs): + update = UP(*args, **kwargs) + update.execute = mock.MagicMock() + self.updates.append(update) + return update + + def setUp(self): + self.exports = kojihub.RootExports() + self.UpdateProcessor = mock.patch('kojihub.kojihub.UpdateProcessor', + side_effect=self.getUpdate).start() + self.updates = [] + self.context = mock.patch('kojihub.kojihub.context').start() + self.context.session.assertLogin = mock.MagicMock() + self.user = {'id': 1, 'name': 'jdoe'} + self.getLoggedInUser = mock.patch.object(self.exports, 'getLoggedInUser', + return_value=self.user).start() + self.get_build = mock.patch('kojihub.kojihub.get_build').start() + self.assert_policy = mock.patch('kojihub.kojihub.assert_policy').start() + self.apply_volume_policy = mock.patch('kojihub.kojihub.apply_volume_policy', + return_value=None).start() + self.move_and_symlink = mock.patch('kojihub.kojihub.move_and_symlink').start() + self.ensure_volume_symlink = mock.patch('kojihub.kojihub.ensure_volume_symlink').start() + self.list_tags = mock.patch('kojihub.kojihub.list_tags', + return_value=[{'id': 101}]).start() + self.set_tag_update = mock.patch('kojihub.kojihub.set_tag_update').start() + self.encode_datetime = mock.patch('kojihub.kojihub.encode_datetime', return_value='NOW').start() + self._now = datetime.datetime.now() + self._datetime = mock.patch('kojihub.kojihub.datetime.datetime').start() + self.now = self._datetime.now = mock.MagicMock(return_value=self._now) + + self.draft_build = { + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'dftrel_1', + 'extra': { + 'draft': { + 'promoted': False, + 'target_release': 'tgtrel_1' + }}, + 'draft': True, + 'volume_id': 99, + 'volume_name': 'X' + } + + self.new_build = { + # no check on the info + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'tgtrel_1', + 'volume_name': 'X' + } + + def tearDown(self): + mock.patch.stopall() + + def test_promote_build_valid(self): + self.get_build.side_effect = [ + self.draft_build, + None, + self.new_build + ] + + extra = json.dumps( + { + 'draft': { + 'promoted': True, + 'target_release': 'tgtrel_1', + 'old_release': 'dftrel_1', + 'promotion_time': 'NOW', + 'promotion_ts': self._now.timestamp(), + 'promoter': self.user['name'] + } + } + ) + + ret = self.exports.promoteBuild('a-draft-build', strict=True) + self.assertEqual(ret, self.new_build) + self.assertEqual(len(self.updates), 1) + update = self.updates[0] + self.assertEqual(update.table, 'build') + self.assertEqual(update.values, self.draft_build) + self.assertEqual(update.data, {'draft': False, + 'release': 'tgtrel_1', + 'extra': extra}) + self.assertEqual(update.rawdata, {}) + self.assertEqual(update.clauses, ['id=%(id)i']) + + def test_promote_build_not_draft(self): + self.get_build.return_value = {'draft': False} + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), "Not a draft build: {'draft': False}") + self.assertEqual(len(self.updates), 0) + + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + + def test_promote_build_no_target_release(self): + draft = { + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'dftrel_1', + 'extra': { + 'draft': { + 'promoted': False + # no target_release + }}, + 'draft': True, + 'volume_id': 99, + 'volume_name': 'X' + } + + self.get_build.return_value = draft + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), f"draft.target_release not found in extra of build: {draft}") + self.assertEqual(len(self.updates), 0) + + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + + def test_promote_build_target_build_exists(self): + old = { + 'id': 'any' + } + self.get_build.side_effect = [self.draft_build, old] + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), f"Target build already exists: {old}") + self.assertEqual(len(self.updates), 0) + self.get_build.assert_called_with({ + 'name': 'foo', + 'version': 'bar', + 'release': 'tgtrel_1' + }) + + self.get_build.reset_mock() + self.get_build.side_effect = [self.draft_build, old] + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + + def test_promote_build_volume_changed(self): + self.get_build.side_effect = [self.draft_build, None] + self.apply_volume_policy.return_value = { + 'id': 100, + 'name': 'Y' + } + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), f"Denial as volume will be changed to Y") + self.assertEqual(len(self.updates), 0) + + self.get_build.reset_mock() + self.get_build.side_effect = [self.draft_build, None] + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) diff --git a/tests/test_hub/test_read_tagged_builds.py b/tests/test_hub/test_read_tagged_builds.py index 96240f6..df82752 100644 --- a/tests/test_hub/test_read_tagged_builds.py +++ b/tests/test_hub/test_read_tagged_builds.py @@ -31,7 +31,7 @@ class TestReadTaggedBuilds(unittest.TestCase): self.tag_name = 'test-tag' self.columns = ['tag.id', 'tag.name', 'build.id', 'build.version', 'build.release', 'build.epoch', 'build.state', 'build.completion_time', 'build.start_time', - 'build.task_id', 'users.id', 'users.name', 'events.id', 'events.time', + 'build.task_id', 'build.draft', 'users.id', 'users.name', 'events.id', 'events.time', 'volume.id', 'volume.name', 'package.id', 'package.name', 'package.name || \'-\' || build.version || \'-\' || build.release', 'tag_listing.create_event'] @@ -40,6 +40,7 @@ class TestReadTaggedBuilds(unittest.TestCase): ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), + ('build.draft', 'draft'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), @@ -54,7 +55,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'volume ON volume.id = build.volume_id', 'users ON users.id = build.owner', ] self.aliases = ['tag_id', 'tag_name', 'id', 'build_id', 'version', 'release', 'epoch', - 'state', 'completion_time', 'start_time', 'task_id', 'owner_id', + 'state', 'completion_time', 'start_time', 'task_id', 'draft', 'owner_id', 'owner_name', 'creation_event_id', 'creation_time', 'volume_id', 'volume_name', 'package_id', 'package_name', 'name', 'nvr', 'create_event'] self.clauses = ['(tag_listing.active = TRUE)', @@ -83,7 +84,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': None + 'type': None, 'draft': 3 } self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, self.joins) @@ -119,7 +120,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': self.pkg_name, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'maven'} + 'type': 'maven', 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -148,7 +149,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'win'} + 'type': 'win', 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -177,7 +178,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'image'} + 'type': 'image', 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -212,10 +213,35 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': type} + 'type': type, 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(self.columns)) self.assertEqual(set(query.aliases), set(self.aliases)) self.assertEqual(set(query.clauses), set(self.clauses)) self.assertEqual(query.values, values) + + def test_get_tagged_builds_draft(self): + self.readPackageList.return_value = self.package_list + kojihub.readTaggedBuilds(self.tag_name, draft=koji.DRAFT_FLAG.DRAFT) + + self.assertEqual(len(self.queries), 1) + query = self.queries[0] + + clauses = copy.deepcopy(self.clauses) + clauses.extend(['draft IS TRUE']) + + values = {'clauses': clauses, 'event': None, 'extra': False, 'fields': self.fields, + 'inherit': False, 'joins': self.joins, 'latest': False, 'owner': None, + 'package': None, 'packages': self.package_list, + 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, + 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], + 'type': None, 'draft': koji.DRAFT_FLAG.DRAFT + } + + self.assertEqual(query.tables, self.tables) + self.assertEqual(query.joins, self.joins) + self.assertEqual(set(query.columns), set(self.columns)) + self.assertEqual(set(query.aliases), set(self.aliases)) + self.assertEqual(set(query.clauses), set(clauses)) + self.assertEqual(query.values, values) diff --git a/tests/test_hub/test_read_tagged_rpms.py b/tests/test_hub/test_read_tagged_rpms.py index 057ceb3..1a32aec 100644 --- a/tests/test_hub/test_read_tagged_rpms.py +++ b/tests/test_hub/test_read_tagged_rpms.py @@ -29,12 +29,13 @@ class TestReadTaggedRPMS(unittest.TestCase): self.readTaggedBuilds = mock.patch('kojihub.kojihub.readTaggedBuilds').start() self.tag_name = 'test-tag' self.columns = ['rpminfo.name', 'rpminfo.version', 'rpminfo.release', 'rpminfo.arch', - 'rpminfo.id', 'rpminfo.epoch', 'rpminfo.payloadhash', 'rpminfo.size', - 'rpminfo.buildtime', 'rpminfo.buildroot_id', 'rpminfo.build_id', - 'rpminfo.metadata_only'] + 'rpminfo.id', 'rpminfo.epoch', 'rpminfo.draft', 'rpminfo.payloadhash', + 'rpminfo.size', 'rpminfo.buildtime', 'rpminfo.buildroot_id', + 'rpminfo.build_id', 'rpminfo.metadata_only'] self.joins = ['tag_listing ON rpminfo.build_id = tag_listing.build_id'] - self.aliases = ['name', 'version', 'release', 'arch', 'id', 'epoch', 'payloadhash', - 'size', 'buildtime', 'buildroot_id', 'build_id', 'metadata_only'] + self.aliases = ['name', 'version', 'release', 'arch', 'id', 'epoch', 'draft', + 'payloadhash', 'size', 'buildtime', 'buildroot_id', 'build_id', + 'metadata_only'] self.clauses = ['(tag_listing.active = TRUE)', 'tag_id=%(tagid)s'] self.tables = ['rpminfo'] @@ -101,3 +102,20 @@ class TestReadTaggedRPMS(unittest.TestCase): self.assertEqual(set(query.aliases), set(aliases)) self.assertEqual(set(query.clauses), set(clauses)) self.assertEqual(query.values, values) + + def test_get_tagged_rpms_draft(self): + self.readTaggedBuilds.return_value = self.build_list + kojihub.readTaggedRPMS(self.tag_name, draft=2, extra=False) + + self.assertEqual(len(self.queries), 1) + query = self.queries[0] + + clauses = copy.deepcopy(self.clauses) + clauses.extend(['rpminfo.draft IS NOT TRUE']) + + self.assertEqual(query.tables, self.tables) + self.assertEqual(set(query.columns), set(self.columns)) + self.assertEqual(set(query.joins), set(self.joins)) + self.assertEqual(set(query.aliases), set(self.aliases)) + self.assertEqual(set(query.clauses), set(clauses)) + self.assertEqual(query.values, {}) \ No newline at end of file diff --git a/tests/test_hub/test_reset_build.py b/tests/test_hub/test_reset_build.py index 13d31b2..06fe49d 100644 --- a/tests/test_hub/test_reset_build.py +++ b/tests/test_hub/test_reset_build.py @@ -44,6 +44,9 @@ class TestResetBuild(unittest.TestCase): self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.context = mock.patch('kojihub.kojihub.context').start() self.context.session.assertPerm = mock.MagicMock() + # don't remove anything unexpected + self.rmtree = mock.patch('koji.util.rmtree').start() + self.unlink = mock.patch('os.unlink').start() self.build_id = 3 self.binfo = {'id': 3, 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'test_nvr', 'nvr': 'test_nvr-3.3-20.el8', 'version': '3.3', 'release': '20', From 3ff5eaadd612447e29b040a4c9af6bb37a377458 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:30 +0000 Subject: [PATCH 2/70] misc fixes --- diff --git a/builder/kojid b/builder/kojid index 3178fd3..e259077 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1120,7 +1120,8 @@ class BuildTask(BaseTaskHandler): # scratch builds do not get imported self.session.host.moveBuildToScratch(self.id, srpm, rpms, logs=logs) else: - self.session.host.completeBuild(self.id, build_id, srpm, rpms, brmap, logs=logs) + self.session.host.completeBuild(self.id, build_id, srpm, rpms, brmap, logs=logs, + draft=opts.get('draft')) except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise @@ -2270,7 +2271,8 @@ class WrapperRPMTask(BaseBuildTask): try: self.session.host.completeBuild(self.id, self.new_build_id, relsrpm, relrpms, brmap, - {'noarch': rellogs}) + {'noarch': rellogs}, + draft=opts.get('draft')) except (SystemExit, ServerExit, KeyboardInterrupt): raise except Exception: diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 81d71e2..351029c 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6308,8 +6308,14 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, binfo = get_build(build_id, strict=True) st_complete = koji.BUILD_STATES['COMPLETE'] st_old = binfo['state'] + koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=binfo) + if binfo.get('draft') != draft: + raise koji.GenericError( + f"draft propetry of build: {binfo['nvr']} is not correct. Expected: {draft}," + f" got {binfo['draft']}" + ) if draft: build['release'] = koji.DRAFT_RELEASE_FORMAT.format(**build) for key in ('name', 'version', 'release', 'epoch', 'task_id'): @@ -6333,7 +6339,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, # now to handle the individual rpms for relpath in [srpm] + rpms: fn = "%s/%s" % (uploadpath, relpath) - rpminfo = import_rpm(fn, binfo, brmap.get(relpath)) + rpminfo = import_rpm(fn, binfo, brmap.get(relpath), draft=draft) import_rpm_file(fn, binfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) if logs: @@ -13702,7 +13708,9 @@ class RootExports(object): ) else: return None - target_build = dslice(binfo, ['name', 'version']) + # task_id and volume_id are for apply_volume_policy + # drop id to get build by NVR + target_build = dslice(binfo, ['name', 'version', 'task_id', 'volume_id']) target_build['release'] = target_release old_build = get_build(target_build) if old_build: @@ -14924,7 +14932,7 @@ class HostExports(object): host.verify() task = Task(task_id) task.assertHost(host.id) - result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs, draft=False) + result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs, draft=draft) build_notification(task_id, build_id) return result diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index 8dc58cf..aeb76eb 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -53,7 +53,8 @@ class TestPromoteBuild(unittest.TestCase): }}, 'draft': True, 'volume_id': 99, - 'volume_name': 'X' + 'volume_name': 'X', + 'task_id': 222 } self.new_build = { @@ -125,7 +126,8 @@ class TestPromoteBuild(unittest.TestCase): }}, 'draft': True, 'volume_id': 99, - 'volume_name': 'X' + 'volume_name': 'X', + 'task_id': 222 } self.get_build.return_value = draft @@ -152,7 +154,9 @@ class TestPromoteBuild(unittest.TestCase): self.get_build.assert_called_with({ 'name': 'foo', 'version': 'bar', - 'release': 'tgtrel_1' + 'release': 'tgtrel_1', + 'task_id': 222, + 'volume_id': 99 }) self.get_build.reset_mock() From af3cfab61db3efb98cf2184c02d60361559c1ad1 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:30 +0000 Subject: [PATCH 3/70] fix rpminfo table --- diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index 0250ad8..c1e6238 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -60,6 +60,7 @@ BEGIN; OR NOT draft); ALTER TABLE rpminfo ADD COLUMN draft BOOLEAN; + UPDATE rpminfo SET draft=FALSE WHERE build_id IS NOT NULL; ALTER TABLE rpminfo DROP CONSTRAINT rpminfo_build_id_fkey; ALTER TABLE rpminfo ADD CONSTRAINT rpminfo_build_id_draft_fkey FOREIGN KEY (build_id, draft) REFERENCES build(id, draft) From 75a1afc794924d1aaebfb87e16e27c4b77f83718 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:31 +0000 Subject: [PATCH 4/70] fix None draft for import_build and new_build --- diff --git a/builder/kojid b/builder/kojid index e259077..de7141e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1103,7 +1103,8 @@ class BuildTask(BaseTaskHandler): # TODO - more pre tests archlist = self.getArchList(build_tag, h, extra=extra_arches) # pass draft option in - data['draft'] = opts.get('draft') + if opts.get('draft'): + data['draft'] = opts.get('draft') # let the system know about the build we're attempting if not self.opts.get('scratch'): # scratch builds do not get imported @@ -1121,7 +1122,7 @@ class BuildTask(BaseTaskHandler): self.session.host.moveBuildToScratch(self.id, srpm, rpms, logs=logs) else: self.session.host.completeBuild(self.id, build_id, srpm, rpms, brmap, logs=logs, - draft=opts.get('draft')) + draft=opts.get('draft') or False) except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise @@ -2180,7 +2181,8 @@ class WrapperRPMTask(BaseBuildTask): if opts.get('custom_user_metadata'): data['extra']['custom_user_metadata'] = opts['custom_user_metadata'] # pass draft option in - data['draft'] = opts.get('draft') + if opts.get('draft'): + data['draft'] = opts.get('draft') self.logger.info("Reading package config for %(name)s" % data) pkg_cfg = self.session.getPackageConfig(build_target['dest_tag'], data['name']) if not opts.get('skip_tag'): @@ -2272,7 +2274,7 @@ class WrapperRPMTask(BaseBuildTask): self.session.host.completeBuild(self.id, self.new_build_id, relsrpm, relrpms, brmap, {'noarch': rellogs}, - draft=opts.get('draft')) + draft=opts.get('draft') or False) except (SystemExit, ServerExit, KeyboardInterrupt): raise except Exception: diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 351029c..87b3601 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6069,6 +6069,8 @@ def new_build(data, strict=False): raise koji.GenericError("No such build extra data: %(extra)r" % data) else: data['extra'] = None + if 'draft' in data: + data['draft'] = bool(data['draft']) # provide a few default values data.setdefault('state', koji.BUILD_STATES['COMPLETE']) @@ -6255,6 +6257,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, convert_value(brmap, cast=dict, check_only=True) convert_value(srpm, cast=str, check_only=True) convert_value(rpms, cast=list, check_only=True) + draft = bool(draft) koji.plugin.run_callbacks('preImport', type='build', srpm=srpm, rpms=rpms, brmap=brmap, task_id=task_id, build_id=build_id, build=None, logs=logs) uploadpath = koji.pathinfo.work() From 8a34f7efa0388f6b8ef3c6911ebfb8e9a134e116 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:32 +0000 Subject: [PATCH 5/70] promote-build command --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 5da1770..c5bd285 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -8161,3 +8161,29 @@ def handle_scheduler_logs(goptions, session, args): for log in logs: print(mask % log) + + +def handle_promote_build(goptions, session, args): + "[misc] Promote a draft build" + usage = "usage: %prog promote-build [options] " + parser = OptionParser(usage=get_usage_str(usage)) + parser.add_option('-f', '--force', action='store_true', default=False, + help='force operation') + (options, args) = parser.parse_args(args) + if len(args) != 1: + parser.error("Please specify a draft build") + draft_build = args[0] + try: + draft_build = int(draft_build) + except ValueError: + pass + activate_session(session, goptions) + if options.force and not session.hasPerm('admin'): + parser.error("--force requires admin privilege") + binfo = session.getBuild(draft_build) + if not binfo: + error("No such build: %s" % draft_build) + if not binfo.get('draft'): + error("Not a draft build: %s" % draft_build) + rinfo = session.promoteBuild(binfo['id'], force=options.force) + print("%s has been promoted to %s" % (binfo['nvr'], rinfo['nvr'])) diff --git a/tests/test_cli/data/list-commands.txt b/tests/test_cli/data/list-commands.txt index ed8a656..e0799eb 100644 --- a/tests/test_cli/data/list-commands.txt +++ b/tests/test_cli/data/list-commands.txt @@ -128,6 +128,7 @@ miscellaneous commands: dist-repo Create a yum repo with distribution options import-comps Import group/package information from a comps file moshimoshi Introduce yourself + promote-build Promote a draft build version Report client and hub versions monitor commands: diff --git a/tests/test_cli/test_promote_build.py b/tests/test_cli/test_promote_build.py new file mode 100644 index 0000000..49530aa --- /dev/null +++ b/tests/test_cli/test_promote_build.py @@ -0,0 +1,103 @@ +from __future__ import absolute_import +import koji +import locale +import mock +import os +import time +from six.moves import StringIO + +from koji_cli.commands import handle_promote_build +from . import utils + + +class TestPromoteBuild(utils.CliTestCase): + def setUp(self): + self.maxDiff = None + self.options = mock.MagicMock() + self.session = mock.MagicMock() + self.buildinfo = {'id': 1, + 'name': 'foo-bar', + 'nvr': 'foo-bar-1.1-11', + 'package_id': 2, + 'package_name': 'test-rpm', + 'release': '11#draft_1', + 'version': '1.1', + 'draft': True} + self.target_binfo = self.buildinfo.copy() + self.target_binfo['relesae'] = '11' + self.target_binfo['draft'] = False + self.error_format = """Usage: %s promote-build [options] +(Specify the --help global option for a list of other help options) + +%s: error: {message} +""" % (self.progname, self.progname) + + @mock.patch('sys.stdout', new_callable=StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_promote_build_valid(self, activate_session, stdout): + build_nvr = 'foo-bar-1.1-11' + self.session.getBuild.return_value = self.buildinfo + self.session.promoteBuild.return_value = self.target_binfo + expected_output = "foo-bar-1.1-11 has been promoted to foo-bar-1.1-11\n" + handle_promote_build(self.options, self.session, [build_nvr]) + self.assert_console_message(stdout, expected_output) + activate_session.assert_called_once_with(self.session, self.options) + self.session.getBuild.assert_called_once_with(build_nvr) + self.session.promoteBuild.assert_called_once_with(self.buildinfo['id'], force=False) + + def test_handle_promote_build_non_exist_build(self): + build_nvr = 'foo-bar-1.1-11' + self.session.getBuild.return_value = None + expected = "No such build: %s\n" % build_nvr + self.assert_system_exit( + handle_promote_build, + self.options, self.session, [build_nvr], + stdout='', + stderr=expected, + exit_code=1) + + def test_handle_promote_build_not_draft(self): + build_nvr = 'foo-bar-1.1-11' + self.session.getBuild.return_value = self.target_binfo + expected = "Not a draft build: %s\n" % build_nvr + self.assert_system_exit( + handle_promote_build, + self.options, self.session, [build_nvr], + stdout='', + stderr=expected, + exit_code=1) + + def test_promote_build_force_not_admin(self): + arguments = ['--force', 'build'] + self.session.hasPerm.return_value = False + self.assert_system_exit( + handle_promote_build, + self.options, self.session, arguments, + stdout='', + stderr=self.format_error_message("--force requires admin privilege"), + exit_code=2) + self.session.getBuild.assert_not_called() + self.session.promoteBuild.assert_not_called() + + def test_promote_build_without_option(self): + arguments = [] + self.assert_system_exit( + handle_promote_build, + self.options, self.session, arguments, + stdout='', + stderr=self.format_error_message("Please specify a draft build"), + exit_code=2, + activate_session=None) + self.session.getBuild.assert_not_called() + self.session.promoteBuild.assert_not_called() + + def test_promote_build_help(self): + self.assert_help( + handle_promote_build, + """Usage: %s promote-build [options] +(Specify the --help global option for a list of other help options) + +Options: + -h, --help show this help message and exit + -f, --force force operation +""" % self.progname) From 77a06c98b3f0ad5e285117c753db9c58955cbcf9 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:33 +0000 Subject: [PATCH 6/70] draft filter opt: use bool/None instead of bit flag --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index c5bd285..4b29c3b 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -2847,9 +2847,9 @@ def anon_handle_list_tagged(goptions, session, args): if options.type: opts['type'] = options.type elif options.no_draft: - opts['draft'] = koji.FLAG_REGULAR_BUILD + opts['draft'] = False elif options.draft_only: - opts['draft'] = koji.FLAG_DRAFT_BUILD + opts['draft'] = True event = koji.util.eventFromOpts(session, options) event_id = None if event: @@ -3526,9 +3526,9 @@ def anon_handle_list_builds(goptions, session, args): if options.no_draft and options.draft_only: parser.error("--draft-only conflits with --no-draft") elif options.no_draft: - opts['draft'] = koji.FLAG_REGULAR_BUILD + opts['draft'] = False elif options.draft_only: - opts['draft'] = koji.FLAG_DRAFT_BUILD + opts['draft'] = True if options.cg: opts['cgID'] = options.cg if options.package: diff --git a/koji/__init__.py b/koji/__init__.py index b0b372e..38f7906 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -27,7 +27,6 @@ from __future__ import absolute_import, division import base64 import datetime import errno -import functools import hashlib import json import logging @@ -300,41 +299,6 @@ DEFAULT_AUTH_TIMEOUT = 60 # draft release format DRAFT_RELEASE_FORMAT = '{release}#draft_{id}' -FLAG_DRAFT_BUILD = 1 -FLAG_REGULAR_BUILD = 2 -FLAG_ALL_BUILD = FLAG_DRAFT_BUILD | FLAG_REGULAR_BUILD - -if six.PY3: - from enum import IntFlag - - # draft build bit FLAGs - class DRAFT_FLAG(IntFlag): - - DRAFT = FLAG_DRAFT_BUILD - REGULAR = FLAG_REGULAR_BUILD - ALL = FLAG_ALL_BUILD - - @classmethod - def _missing_(cls, value): - if not isinstance(value, int): - raise ValueError("%r is not a valid %s" % (value, cls.__name__)) - # diable member creation and negative integer - return cls._value2member_map_.get(value) - - def convert_draft_option(func=None, kw='draft'): - def wrapper(func): - @functools.wraps(func) - def convert(*args, **kwargs): - if kw in kwargs: - kwargs[kw] = DRAFT_FLAG(kwargs[kw]) - return func(*args, **kwargs) - return convert - if func is None: - return wrapper - else: - return wrapper(func) - - # BEGIN kojikamid dup # # Exceptions diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 87b3601..b52c896 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -55,7 +55,6 @@ import rpm from psycopg2._psycopg import IntegrityError import koji -from koji import DRAFT_FLAG, convert_draft_option import koji.plugin import koji.policy import koji.rpmdiff @@ -1395,9 +1394,8 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None, pattern=None return query.iterate() -@convert_draft_option def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, - type=None, extra=False, *, draft=DRAFT_FLAG.ALL): + type=None, extra=False, *, draft=None): """Returns a list of builds for specified tag :param int tag: tag ID @@ -1409,10 +1407,10 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, :param str type: restrict the list to builds of the given type. Currently the supported types are 'maven', 'win', 'image', or any custom content generator btypes. :param bool extra: Set to "True" to get the build extra info - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular builds + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only :returns [dict]: list of buildinfo dicts """ # build - id pkg_id version release epoch @@ -1522,9 +1520,8 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, return builds -@convert_draft_option def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, latest=True, - rpmsigs=False, owner=None, type=None, extra=True, *, draft=DRAFT_FLAG.ALL): + rpmsigs=False, owner=None, type=None, extra=True, draft=None): """Returns a list of rpms and builds for specified tag :param int|str tag: The tag name or ID to search @@ -1544,10 +1541,10 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late :param str type: Filter by build type. Supported types are 'maven', 'win', and 'image'. :param bool extra: Set to "False" to skip the rpm extra info - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular builds + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only :returns: a two-element list. The first element is the list of RPMs, and the second element is the list of builds. """ @@ -4708,9 +4705,8 @@ def get_rpm(rpminfo, strict=False, multi=False, build=None): return ret -@convert_draft_option def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, - arches=None, queryOpts=None, *, draft=DRAFT_FLAG.ALL): + arches=None, queryOpts=None, draft=None): """List RPMS. If buildID, imageID and/or buildrootID are specified, restrict the list of RPMs to only those RPMs that are part of that build, or were built in that buildroot. If componentBuildrootID is specified, @@ -4743,11 +4739,11 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID If no build has the given ID, or the build generated no RPMs, an empty list is returned. - The option draft with a bit flag(enum.IntFlag) value is to filter rpm by that - rpm belongs to a draft build, a regular build or both (default). - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular + The option draft with a bool/None value is to filter rpm by that + rpm belongs to a draft build, a regular build or both (default). It stands for: + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only """ fields = [('rpminfo.id', 'id'), ('rpminfo.name', 'name'), ('rpminfo.version', 'version'), @@ -8501,8 +8497,7 @@ def query_history(tables=None, **kwargs): return ret -@convert_draft_option -def untagged_builds(name=None, queryOpts=None, *, draft=DRAFT_FLAG.ALL): +def untagged_builds(name=None, queryOpts=None, *, draft=None): """Returns the list of untagged builds""" st_complete = koji.BUILD_STATES['COMPLETE'] # following can be achieved with simple query but with @@ -11884,9 +11879,8 @@ class RootExports(object): raise koji.GenericError("Finished task's priority can't be updated") task.setPriority(priority, recurse=recurse) - @convert_draft_option def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, - owner=None, type=None, strict=True, extra=False, *, draft=DRAFT_FLAG.ALL): + owner=None, type=None, strict=True, extra=False, draft=None): """List builds tagged with tag. :param int|str tag: tag name or ID number @@ -11902,10 +11896,10 @@ class RootExports(object): :param bool strict: If tag doesn't exist, an exception is raised, unless strict is False in which case returns an empty list. :param bool extra: Set to "True" to get the build extra info - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular builds + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only """ # lookup tag id tag = get_tag(tag, strict=strict, event=event) @@ -11920,10 +11914,9 @@ class RootExports(object): if build['package_name'].lower().startswith(prefix)] return results - @convert_draft_option def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, rpmsigs=False, owner=None, type=None, strict=True, extra=True, - *, draft=DRAFT_FLAG.ALL): + draft=None): """List rpms and builds within tag. :param int|str tag: tag name or ID number @@ -11942,10 +11935,10 @@ class RootExports(object): :param bool strict: If tag doesn't exist, an exception is raised, unless strict is False in which case returns an empty list. :param bool extra: Set to "False" to skip the rpms extra info - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only """ # lookup tag id tag = get_tag(tag, strict=strict, event=event) @@ -11978,11 +11971,10 @@ class RootExports(object): return readTaggedArchives(tag['id'], event=event, inherit=inherit, latest=latest, package=package, type=type, extra=extra) - @convert_draft_option def listBuilds(self, packageID=None, userID=None, taskID=None, prefix=None, state=None, volumeID=None, source=None, createdBefore=None, createdAfter=None, completeBefore=None, completeAfter=None, type=None, typeInfo=None, - queryOpts=None, pattern=None, cgID=None, *, draft=DRAFT_FLAG.ALL): + queryOpts=None, pattern=None, cgID=None, draft=None): """ Return a list of builds that match the given parameters @@ -12018,10 +12010,10 @@ class RootExports(object): fields are matched For type=win, the provided platform fields are matched - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular builds + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only :returns: Returns a list of maps. Each map contains the following keys: @@ -12182,8 +12174,7 @@ class RootExports(object): return query.iterate() - @convert_draft_option - def getLatestBuilds(self, tag, event=None, package=None, type=None, *, draft=DRAFT_FLAG.ALL): + def getLatestBuilds(self, tag, event=None, package=None, type=None, draft=None): """List latest builds for tag (inheritance enabled, wrapper of readTaggedBuilds) :param int tag: tag ID @@ -12191,10 +12182,10 @@ class RootExports(object): :param int package: filter on package name :param str type: restrict the list to builds of the given type. Currently the supported types are 'maven', 'win', 'image', or any custom content generator btypes. - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular builds + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only :returns [dict]: list of buildinfo dicts """ @@ -12204,9 +12195,8 @@ class RootExports(object): return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type, draft=draft) - @convert_draft_option def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None, - *, draft=DRAFT_FLAG.ALL): + draft=None): """List latest RPMS for tag (inheritance enabled, wrapper of readTaggedBuilds) :param int|str tag: The tag name or ID to search @@ -12219,10 +12209,10 @@ class RootExports(object): :param bool rpmsigs: query will return one record per rpm/signature combination :param str type: Filter by build type. Supported types are 'maven', 'win', and 'image'. - :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates - - DRAFT(1): draft only - - REGULAR(2): regular only - - ALL(3): both draft and regular + :param bool draft: bool or None option that indicates the filter based on draft field + - None: no filter (both draft and regular builds) + - True: draft only + - False: regular only :returns: a two-element list. The first element is the list of RPMs, and the second element is the list of builds. """ @@ -16027,27 +16017,24 @@ def reject_draft(buildinfo, error=None): def append_draft_clause(draft, clauses, table=None): - """append proper clause in build/rpm query for draft flags + """append proper clause in build/rpm query for draft option - DRAFT=1: append "draft IS True" - REGULAR=2: append "draft IS NOT True" - ALL(DRAFT|REGULAR)=3: do nothing - - :param DRAFT_FLAGS draft: draft bit flag(s) + :param bool draft: draft option: + True: append "draft IS True" + False: append "draft IS NOT True" + None: do nothing :param list clauses: clauses list to construct query by QueryProcessor, which the draft clause to append """ - if not isinstance(draft, (DRAFT_FLAG)): - raise koji.ParameterError(f'draft must be a DRAFT_FLAG, but got {draft}') if not table: table = '' else: table += '.' - if DRAFT_FLAG.ALL in draft: + if draft is None: return - if DRAFT_FLAG.DRAFT in draft: + if draft: clauses.append(f'{table}draft IS TRUE') - if DRAFT_FLAG.REGULAR in draft: + else: # null is included clauses.append(f'{table}draft IS NOT TRUE') diff --git a/tests/test_cli/test_list_tagged.py b/tests/test_cli/test_list_tagged.py index 3bccfe1..a009fbd 100644 --- a/tests/test_cli/test_list_tagged.py +++ b/tests/test_cli/test_list_tagged.py @@ -103,7 +103,7 @@ n-v-r tag owner self.session.getTag.assert_called_once_with(self.tag, event=self.event_id) self.session.listTagged.assert_called_once_with( self.tag, event=self.event_id, inherit=True, latest=True, package=self.pkg, - draft=2) + draft=False) self.session.listTaggedRPMS.assert_not_called() self.assert_console_message(stdout, expected) @@ -121,7 +121,7 @@ n-v-r tag owner self.ensure_connection_mock.assert_called_once_with(self.session, self.options) self.session.getTag.assert_called_once_with(self.tag, event=None) self.session.listTagged.assert_called_once_with( - self.tag, inherit=True, latest=True, package=self.pkg, draft=1) + self.tag, inherit=True, latest=True, package=self.pkg, draft=True) self.session.listTaggedRPMS.assert_not_called() @mock.patch('sys.stdout', new_callable=six.StringIO) diff --git a/tests/test_hub/test_list_builds.py b/tests/test_hub/test_list_builds.py index fd80bac..356ca2d 100644 --- a/tests/test_hub/test_list_builds.py +++ b/tests/test_hub/test_list_builds.py @@ -101,7 +101,7 @@ class TestListBuilds(unittest.TestCase): package_id = 1 self.get_package_id.return_value = package_id self.query_executeOne.return_value = None - self.exports.listBuilds(packageID=package, draft=1) + self.exports.listBuilds(packageID=package, draft=True) self.assertEqual(len(self.queries), 1) args, kwargs = self.QueryProcessor.call_args qp = QP(**kwargs) diff --git a/tests/test_hub/test_read_tagged_builds.py b/tests/test_hub/test_read_tagged_builds.py index df82752..f919df0 100644 --- a/tests/test_hub/test_read_tagged_builds.py +++ b/tests/test_hub/test_read_tagged_builds.py @@ -84,7 +84,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': None, 'draft': 3 + 'type': None, 'draft': None } self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, self.joins) @@ -120,7 +120,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': self.pkg_name, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'maven', 'draft': 3} + 'type': 'maven', 'draft': None} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -149,7 +149,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'win', 'draft': 3} + 'type': 'win', 'draft': None} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -178,7 +178,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'image', 'draft': 3} + 'type': 'image', 'draft': None} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -213,7 +213,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': type, 'draft': 3} + 'type': type, 'draft': None} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(self.columns)) @@ -223,7 +223,7 @@ class TestReadTaggedBuilds(unittest.TestCase): def test_get_tagged_builds_draft(self): self.readPackageList.return_value = self.package_list - kojihub.readTaggedBuilds(self.tag_name, draft=koji.DRAFT_FLAG.DRAFT) + kojihub.readTaggedBuilds(self.tag_name, draft=True) self.assertEqual(len(self.queries), 1) query = self.queries[0] @@ -236,7 +236,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': None, 'draft': koji.DRAFT_FLAG.DRAFT + 'type': None, 'draft': True } self.assertEqual(query.tables, self.tables) diff --git a/tests/test_hub/test_read_tagged_rpms.py b/tests/test_hub/test_read_tagged_rpms.py index 1a32aec..a39f996 100644 --- a/tests/test_hub/test_read_tagged_rpms.py +++ b/tests/test_hub/test_read_tagged_rpms.py @@ -105,7 +105,7 @@ class TestReadTaggedRPMS(unittest.TestCase): def test_get_tagged_rpms_draft(self): self.readTaggedBuilds.return_value = self.build_list - kojihub.readTaggedRPMS(self.tag_name, draft=2, extra=False) + kojihub.readTaggedRPMS(self.tag_name, draft=False, extra=False) self.assertEqual(len(self.queries), 1) query = self.queries[0] From 241daabecd05a694d92fa8d215ab7b5e483650fe Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:33 +0000 Subject: [PATCH 7/70] remove unused methods in PathInfo --- diff --git a/koji/__init__.py b/koji/__init__.py index 38f7906..19f565a 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2367,39 +2367,6 @@ class PathInfo(object): return self.volumedir(build.get('volume_name')) + \ ("/packages/%(name)s/%(version)s/%(release)s" % build) - def to_buildinfo(self, path): - """Revert build dir path (/[/]packages///) - to build map with the data below: - - - name - - version - - release - - volume_name - """ - path = path.rstrip('/') - parts = path.rsplit('/', 4) - if len(parts) != 5: - raise GenericError("Invalid build path: %s" % path) - if parts[-4] != 'packages': - raise GenericError("Invalid build path: %s, packages subdir not found" % path) - nvr = parts[-3:] - rest = parts[0] - if not rest.startswith(self.topdir): - raise GenericError("Invalid build path: %s, topdir is not %r" % (path, self.topdir)) - vol_part = rest[len(self.topdir):] - if not vol_part: - vol = 'DEFAULT' - elif vol_part.startswith('/vol/'): - vol = vol_part[5:] - else: - raise GenericError( - "Invalid build path: %s, volume dir: %s is incorrect" % (path, vol_part) - ) - return {'name': nvr[0], - 'version': nvr[1], - 'release': nvr[2], - 'volume_name': vol} - def mavenbuild(self, build): """Return the directory where the Maven build exists in the global store (/mnt/koji/packages)""" @@ -2447,27 +2414,6 @@ class PathInfo(object): """Return the path (relative to build_dir) where an rpm belongs""" return "%(arch)s/%(name)s-%(version)s-%(release)s.%(arch)s.rpm" % rpminfo - def to_rpminfo(self, path, full=False): - """Revert rpm path in build dir to rpm nvra map with/without build dir""" - if path.endswith('/'): - raise GenericError("Invalid path: %s, cannot be a directory" % path) - parts = path.rsplit('/', 2) - if full: - if len(parts) != 3: - raise GenericError("No build dir in path: %s" % path) - return self.to_buildinfo(parts[-3]), self._to_rpminfo(parts[-2:]) - return self._to_rpminfo(parts) - - def _to_rpminfo(self, parts): - if len(parts) != 2 or not parts[0]: - raise GenericError("Invalid path: %s" % '/'.join(parts)) - rpminfo = parse_NVRA(parts[-1]) - if parts[0] != rpminfo['arch']: - raise GenericError( - 'mismatch between arch dir (%s) and arch (%s) in rpm' % (parts[0], rpminfo) - ) - return rpminfo - def signed(self, rpminfo, sigkey): """Return the path (relative to build dir) where a signed rpm lives""" return "data/signed/%s/" % sigkey + self.rpm(rpminfo) From 70518c01a1547b7383b2f4c06eb9a07a5cf1119d Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:33 +0000 Subject: [PATCH 8/70] remove unused yesno helper --- diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index b3a0812..c605940 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -925,10 +925,3 @@ class DatetimeJSONEncoder(json.JSONEncoder): if isinstance(o, xmlrpc_client.DateTime): return str(o) return json.JSONEncoder.default(self, o) - - -def yesno(x): - if x: - return 'Y' - else: - return 'N' From c045bc7100c5193e4e9eb09c9f9f4be233519e0e Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:34 +0000 Subject: [PATCH 9/70] promoteBuild: split main function to _promote_build --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index b52c896..6ef7039 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -13679,94 +13679,7 @@ class RootExports(object): :rtype: dict """ context.session.assertLogin() - user = self.getLoggedInUser() - - binfo = get_build(build, strict=strict) - if not binfo: - return None - if not binfo.get('draft'): - if strict: - raise koji.GenericError(f'Not a draft build: {binfo}') - else: - return None - old_release = binfo['release'] - extra = binfo.get('extra') or {} - # get target release in binfo.extra - draft_info = extra.get('draft', {}).copy() - target_release = draft_info.get('target_release') - if not target_release: - if strict: - raise koji.GenericError( - f'draft.target_release not found in extra of build: {binfo}' - ) - else: - return None - # task_id and volume_id are for apply_volume_policy - # drop id to get build by NVR - target_build = dslice(binfo, ['name', 'version', 'task_id', 'volume_id']) - target_build['release'] = target_release - old_build = get_build(target_build) - if old_build: - if strict: - raise koji.GenericError(f'Target build already exists: {old_build}') - else: - return None - # policy checks - policy_data = { - 'build': binfo['id'], - 'target_release': target_release - } - assert_policy('draft_promotion', policy_data, force=force) - # volume check, deny it if volume is changed as it's only allowed for admin - # after building, see applyVolumePolicy - new_volume = apply_volume_policy(target_build, strict=False, dry_run=True) - if new_volume is not None and new_volume['id'] != binfo['volume_id']: - # probably we can just apply the volume change here - if strict: - raise koji.GenericError( - f'Denial as volume will be changed to {new_volume["name"]}' - ) - else: - return None - - koji.plugin.run_callbacks( - 'preBuildPromote', - draft_release=old_release, - target_release=target_release, - build=binfo, - user=user - ) - - # temp solution with python generated time, maybe better to be an event? - now = datetime.datetime.now() - - extra['draft'] = draft_info - draft_info['promoted'] = True - draft_info['old_release'] = old_release - draft_info['promotion_time'] = encode_datetime(now) - draft_info['promotion_ts'] = now.timestamp() - draft_info['promoter'] = user['name'] - extra = json.dumps(extra) - - update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) - update.set(draft=False, release=target_release, extra=extra) - update.execute() - - new_binfo = get_build(binfo['id'], strict=strict) - move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) - ensure_volume_symlink(new_binfo) - - for tag in list_tags(build=binfo['id']): - set_tag_update(tag['id'], 'DRAFT_PROMOTION') - - koji.plugin.run_callbacks( - 'postBuildPromote', - draft_release=old_release, - target_release=target_release, - build=new_binfo, - user=user - ) - return new_binfo + return _promote_build(build, strict=strict, force=force) def count(self, methodName, *args, **kw): """Execute the XML-RPC method with the given name and count the results. @@ -16056,3 +15969,99 @@ def _clean_draft_link(promoted_build): # only unlink whe its a symlink. if os.path.islink(draft_builddir): os.unlink(draft_builddir) + + +def _promote_build(build, user=None, strict=True, force=False): + """promote a draft build to a regular one""" + binfo = get_build(build, strict=strict) + if not binfo: + return None + if not binfo.get('draft'): + if strict: + raise koji.GenericError(f'Not a draft build: {binfo}') + else: + return None + old_release = binfo['release'] + extra = binfo.get('extra') or {} + # get target release in binfo.extra + draft_info = extra.get('draft', {}).copy() + target_release = draft_info.get('target_release') + if not target_release: + if strict: + raise koji.GenericError( + f'draft.target_release not found in extra of build: {binfo}' + ) + else: + return None + # task_id and volume_id are for apply_volume_policy + # drop id to get build by NVR + target_build = dslice(binfo, ['name', 'version', 'task_id', 'volume_id']) + target_build['release'] = target_release + old_build = get_build(target_build) + if old_build: + if strict: + raise koji.GenericError(f'Target build already exists: {old_build}') + else: + return None + + user = get_user(user, strict=strict) + if not user and not strict: + return None + + # policy checks + policy_data = { + 'build': binfo['id'], + 'target_release': target_release, + 'user_id': user['id'] + } + assert_policy('draft_promotion', policy_data, force=force) + # volume check, deny it if volume is changed as it's only allowed for admin + # after building, see applyVolumePolicy + new_volume = apply_volume_policy(target_build, strict=False, dry_run=True) + if new_volume is not None and new_volume['id'] != binfo['volume_id']: + # probably we can just apply the volume change here + if strict: + raise koji.GenericError( + f'Denial as volume will be changed to {new_volume["name"]}' + ) + else: + return None + + koji.plugin.run_callbacks( + 'preBuildPromote', + draft_release=old_release, + target_release=target_release, + build=binfo, + user=user + ) + + # temp solution with python generated time, maybe better to be an event? + now = datetime.datetime.now() + + extra['draft'] = draft_info + draft_info['promoted'] = True + draft_info['old_release'] = old_release + draft_info['promotion_time'] = encode_datetime(now) + draft_info['promotion_ts'] = now.timestamp() + draft_info['promoter'] = user['name'] + extra = json.dumps(extra) + + update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) + update.set(draft=False, release=target_release, extra=extra) + update.execute() + + new_binfo = get_build(binfo['id'], strict=strict) + move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) + ensure_volume_symlink(new_binfo) + + for tag in list_tags(build=binfo['id']): + set_tag_update(tag['id'], 'DRAFT_PROMOTION') + + koji.plugin.run_callbacks( + 'postBuildPromote', + draft_release=old_release, + target_release=target_release, + build=new_binfo, + user=user + ) + return new_binfo diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index aeb76eb..ba9b12d 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -25,8 +25,7 @@ class TestPromoteBuild(unittest.TestCase): self.context = mock.patch('kojihub.kojihub.context').start() self.context.session.assertLogin = mock.MagicMock() self.user = {'id': 1, 'name': 'jdoe'} - self.getLoggedInUser = mock.patch.object(self.exports, 'getLoggedInUser', - return_value=self.user).start() + self.get_user = mock.patch('kojihub.kojihub.get_user', return_value=self.user).start() self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.assert_policy = mock.patch('kojihub.kojihub.assert_policy').start() self.apply_volume_policy = mock.patch('kojihub.kojihub.apply_volume_policy', From 6fe42b6212d6e45c8b304d599b1cc7ef0d256b46 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:35 +0000 Subject: [PATCH 10/70] cil wrapper-rpm: input check and more reasonable opt --create-draft --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 4b29c3b..2b63876 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -834,8 +834,8 @@ def handle_wrapper_rpm(options, session, args): parser.add_option("--nowait", action="store_false", dest="wait", help="Don't wait on build") parser.add_option("--background", action="store_true", help="Run the build at a lower priority") - parser.add_option("--draft", action="store_true", - help="Build draft build instead") + parser.add_option("--create-draft", action="store_true", + help="Create a new draft build instead") (build_opts, args) = parser.parse_args(args) if build_opts.inis: @@ -845,8 +845,12 @@ def handle_wrapper_rpm(options, session, args): if len(args) < 3: parser.error("You must provide a build target, a build ID or NVR, " "and a SCM URL to a specfile fragment") - if build_opts.scratch and build_opts.draft: - parser.error("--scratch and --draft cannot be both specfied") + if build_opts.create_draft: + print("Will create a draft build instead") + build_opts.create_build = True + if build_opts.scratch: + # TODO: --scratch and --create-build conflict too + parser.error("--scratch and --create-draft cannot be both specfied") activate_session(session, options) target = args[0] @@ -882,7 +886,7 @@ def handle_wrapper_rpm(options, session, args): opts['skip_tag'] = True if build_opts.scratch: opts['scratch'] = True - if build_opts.draft: + if build_opts.create_draft: opts['draft'] = True task_id = session.wrapperRPM(build_id, url, target, priority, opts=opts) print("Created task: %d" % task_id) diff --git a/tests/test_cli/test_wrapper_rpm.py b/tests/test_cli/test_wrapper_rpm.py index 94fbd88..507b5d4 100644 --- a/tests/test_cli/test_wrapper_rpm.py +++ b/tests/test_cli/test_wrapper_rpm.py @@ -196,7 +196,7 @@ class TestWrapperRpm(utils.CliTestCase): @mock.patch('koji_cli.commands.activate_session') def test_handle_wrapper_rpm_argument_error( self, activate_session_mock, stderr, stdout): - """Test handle_wrapper_rpm help message output""" + """Test handle_wrapper_rpm error message output""" arguments = [] options = mock.MagicMock() @@ -219,8 +219,35 @@ class TestWrapperRpm(utils.CliTestCase): # Finally, assert that things were called as we expected. activate_session_mock.assert_not_called() + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_wrapper_rpm_argument_conflict_error( + self, activate_session_mock, stderr, stdout): + """Test handle_wrapper_rpm error message output""" + arguments = ['--scratch', '--create-draft', 'foo', 'n-v-r', 'scmurl'] + options = mock.MagicMock() + + # Mock out the xmlrpc server + session = mock.MagicMock() + + # Run it and check immediate output + expected = self.format_error_message( + "--scratch and --create-draft cannot be both specfied") + self.assert_system_exit( + handle_wrapper_rpm, + options, + session, + arguments, + stdout='Will create a draft build instead\n', + stderr=expected, + activate_session=None) + + # Finally, assert that things were called as we expected. + activate_session_mock.assert_not_called() + def test_handle_wrapper_rpm_help(self): - """Test handle_wrapper_rpm help message output""" + """Test handle_wrapper_rpm help message output""" self.assert_help( handle_wrapper_rpm, """Usage: %s wrapper-rpm [options] @@ -237,7 +264,7 @@ Options: --wait Wait on build, even if running in the background --nowait Don't wait on build --background Run the build at a lower priority - --draft Build draft build instead + --create-draft Create a new draft build instead """ % self.progname) From 44d59ecbc77c881a723f5935e7a307b23441888c Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:35 +0000 Subject: [PATCH 11/70] can only promote COMPLETE draft build --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 6ef7039..ff45dca 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -15981,6 +15981,13 @@ def _promote_build(build, user=None, strict=True, force=False): raise koji.GenericError(f'Not a draft build: {binfo}') else: return None + state = koji.BUILD_STATES[binfo['state']] + if state != 'COMPLETE': + if strict: + raise koji.GenericError(f"Cannot promote build - {binfo['nvr']}." + f' Reason: state ({state}) is not COMPLETE.') + else: + return None old_release = binfo['release'] extra = binfo.get('extra') or {} # get target release in binfo.extra diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index ba9b12d..d987a80 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -45,11 +45,13 @@ class TestPromoteBuild(unittest.TestCase): 'name': 'foo', 'version': 'bar', 'release': 'dftrel_1', + 'nvr': 'testnvr', 'extra': { 'draft': { 'promoted': False, 'target_release': 'tgtrel_1' }}, + 'state': 1, 'draft': True, 'volume_id': 99, 'volume_name': 'X', @@ -123,6 +125,7 @@ class TestPromoteBuild(unittest.TestCase): 'promoted': False # no target_release }}, + 'state': 1, 'draft': True, 'volume_id': 99, 'volume_name': 'X', @@ -140,6 +143,39 @@ class TestPromoteBuild(unittest.TestCase): self.assertIsNone(ret) self.assertEqual(len(self.updates), 0) + def test_promote_build_not_completed(self): + draft = { + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'dftrel_1', + 'nvr': 'testnvr', + 'extra': { + 'draft': { + 'promoted': False + # no target_release + }}, + 'draft': True, + 'state': 0, + 'volume_id': 99, + 'volume_name': 'X', + 'task_id': 222 + } + + self.get_build.return_value = draft + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual( + str(cm.exception), + f"Cannot promote build - {draft['nvr']}. Reason: state (BUILDING) is not COMPLETE." + ) + self.assertEqual(len(self.updates), 0) + + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + def test_promote_build_target_build_exists(self): old = { 'id': 'any' From 0859f1ba896ba0718780a8bd0429a5906eb18900 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:36 +0000 Subject: [PATCH 12/70] promoteBuild: update volume based on volume policy --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index ff45dca..d87ac91 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -16000,9 +16000,8 @@ def _promote_build(build, user=None, strict=True, force=False): ) else: return None - # task_id and volume_id are for apply_volume_policy # drop id to get build by NVR - target_build = dslice(binfo, ['name', 'version', 'task_id', 'volume_id']) + target_build = dslice(binfo, ['name', 'version']) target_build['release'] = target_release old_build = get_build(target_build) if old_build: @@ -16022,17 +16021,6 @@ def _promote_build(build, user=None, strict=True, force=False): 'user_id': user['id'] } assert_policy('draft_promotion', policy_data, force=force) - # volume check, deny it if volume is changed as it's only allowed for admin - # after building, see applyVolumePolicy - new_volume = apply_volume_policy(target_build, strict=False, dry_run=True) - if new_volume is not None and new_volume['id'] != binfo['volume_id']: - # probably we can just apply the volume change here - if strict: - raise koji.GenericError( - f'Denial as volume will be changed to {new_volume["name"]}' - ) - else: - return None koji.plugin.run_callbacks( 'preBuildPromote', @@ -16061,6 +16049,11 @@ def _promote_build(build, user=None, strict=True, force=False): move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) ensure_volume_symlink(new_binfo) + # apply volume policy in case it's changed by release update. + apply_volume_policy(new_binfo, strict=False) + + # adding DRAFT_PROMOTION for kojira, + # as the latest promoted build should be that latest one. for tag in list_tags(build=binfo['id']): set_tag_update(tag['id'], 'DRAFT_PROMOTION') diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index d987a80..7d7b87a 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -2,6 +2,7 @@ import datetime import json import mock import unittest + import koji import kojihub @@ -101,6 +102,9 @@ class TestPromoteBuild(unittest.TestCase): 'extra': extra}) self.assertEqual(update.rawdata, {}) self.assertEqual(update.clauses, ['id=%(id)i']) + self.apply_volume_policy.assert_called_once_with( + self.new_build, strict=False + ) def test_promote_build_not_draft(self): self.get_build.return_value = {'draft': False} @@ -189,9 +193,7 @@ class TestPromoteBuild(unittest.TestCase): self.get_build.assert_called_with({ 'name': 'foo', 'version': 'bar', - 'release': 'tgtrel_1', - 'task_id': 222, - 'volume_id': 99 + 'release': 'tgtrel_1' }) self.get_build.reset_mock() @@ -199,20 +201,3 @@ class TestPromoteBuild(unittest.TestCase): ret = self.exports.promoteBuild('a-regular-build', strict=False) self.assertIsNone(ret) self.assertEqual(len(self.updates), 0) - - def test_promote_build_volume_changed(self): - self.get_build.side_effect = [self.draft_build, None] - self.apply_volume_policy.return_value = { - 'id': 100, - 'name': 'Y' - } - with self.assertRaises(koji.GenericError) as cm: - self.exports.promoteBuild('a-regular-build', strict=True) - self.assertEqual(str(cm.exception), f"Denial as volume will be changed to Y") - self.assertEqual(len(self.updates), 0) - - self.get_build.reset_mock() - self.get_build.side_effect = [self.draft_build, None] - ret = self.exports.promoteBuild('a-regular-build', strict=False) - self.assertIsNone(ret) - self.assertEqual(len(self.updates), 0) From 74ba6c947ff0a9db6e9d0d287e4670d105187ee6 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:36 +0000 Subject: [PATCH 13/70] cli rpminfo: handle no srpm case --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 2b63876..c82756e 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -3664,11 +3664,14 @@ def anon_handle_rpminfo(goptions, session, args): if info['arch'] == 'src': srpminfo = info.copy() else: - srpminfo = session.listRPMs(buildID=info['build_id'], arches='src')[0] - if srpminfo['epoch'] is None: - srpminfo['epoch'] = "" - else: - srpminfo['epoch'] = str(srpminfo['epoch']) + ":" + srpminfo = None + srpms = session.listRPMs(buildID=info['build_id'], arches='src') + if srpms: + srpminfo = srpms[0] + if srpminfo['epoch'] is None: + srpminfo['epoch'] = "" + else: + srpminfo['epoch'] = str(srpminfo['epoch']) + ":" buildinfo = session.getBuild(info['build_id']) print("RPM: %(epoch)s%(name)s-%(version)s-%(release)s.%(arch)s [%(id)d]" % info) if info.get('draft'): @@ -3681,9 +3684,16 @@ def anon_handle_rpminfo(goptions, session, args): print("Build: %(nvr)s [%(id)d]" % buildinfo) print("RPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info))) - print("SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % srpminfo) - print("SRPM Path: %s" % - os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(srpminfo))) + if srpminfo: + srpm_str = "%(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % srpminfo + srpm_path = os.path.join( + koji.pathinfo.build(buildinfo), + koji.pathinfo.rpm(srpminfo) + ) + else: + srpm_path = srpm_str = "(none)" + print("SRPM: %s" % srpm_str) + print("SRPM Path: %s" % srpm_path) print("Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(info['buildtime']))) print("SIGMD5: %(payloadhash)s" % info) From 2195be9f8d8db076029088ace1ecc1f23b46b8bd Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:37 +0000 Subject: [PATCH 14/70] a comment for CONSTRAINT draft_for_rpminfo --- diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index c1e6238..804134a 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -54,6 +54,7 @@ BEGIN; INSERT INTO permissions (name, description) VALUES ('draft-promoter', 'The permission required in the default "draft_promotion" hub policy rule to promote draft build.'); ALTER TABLE build ADD COLUMN draft BOOLEAN NOT NULL DEFAULT 'false'; + -- required by constraint rpminfo_build_id_draft_fkey on table rpminfo ALTER TABLE build ADD CONSTRAINT draft_for_rpminfo UNIQUE (id, draft); ALTER TABLE build ADD CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) diff --git a/schemas/schema.sql b/schemas/schema.sql index b03b891..5d820a8 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -296,6 +296,7 @@ CREATE TABLE build ( cg_id INTEGER REFERENCES content_generator(id), extra TEXT, CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), + -- required by constraint rpminfo_build_id_draft_fkey on table rpminfo CONSTRAINT draft_for_rpminfo UNIQUE (id, draft), CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR (state <> 0 AND completion_time IS NOT NULL)), From d3ac45ca6071ea1b18b7f8826403fa5c76f44d7d Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:37 +0000 Subject: [PATCH 15/70] move clauses.append out of append_draft_clause --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index d87ac91..0c2d440 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -1483,7 +1483,8 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, clauses.append('package.name = %(package)s') if owner: clauses.append('users.name = %(owner)s') - append_draft_clause(draft, clauses) + if draft is not None: + clauses.append(draft_clause(draft)) queryOpts = {'order': '-create_event'} # latest first if extra: fields.append(('build.extra', 'extra')) @@ -1594,7 +1595,8 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late clauses.append('rpminfo.arch IN %(arch)s') else: raise koji.GenericError('Invalid type for arch option: %s' % builtins.type(arch)) - append_draft_clause(draft, clauses, table='rpminfo') + if draft is not None: + clauses.append(draft_clause(draft, table='rpminfo')) if extra: fields.append(('rpminfo.extra', 'extra')) @@ -4790,7 +4792,8 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID clauses.append('rpminfo.arch = %(arches)s') else: raise koji.GenericError('Invalid type for "arches" parameter: %s' % type(arches)) - append_draft_clause(draft, clauses) + if draft is not None: + clauses.append(draft_clause(draft)) fields, aliases = zip(*fields) query = QueryProcessor(columns=fields, aliases=aliases, @@ -8514,7 +8517,8 @@ def untagged_builds(name=None, queryOpts=None, *, draft=None): if name is not None: clauses.append('package.name = %(name)s') - append_draft_clause(draft, clauses) + if draft is not None: + clauses.append(draft_clause(draft)) query = QueryProcessor(tables=['build', 'package'], columns=['build.id', 'package.name', 'build.version', 'build.release'], @@ -12164,7 +12168,8 @@ class RootExports(object): btype_id = btype['id'] joins.append('build_types ON build.id = build_types.build_id ' 'AND btype_id = %(btype_id)s') - append_draft_clause(draft, clauses) + if draft is not None: + clauses.append(draft_clause(draft)) query = QueryProcessor(columns=[pair[0] for pair in fields], aliases=[pair[1] for pair in fields], @@ -15929,31 +15934,33 @@ def reject_draft(buildinfo, error=None): raise error -def append_draft_clause(draft, clauses, table=None): - """append proper clause in build/rpm query for draft option +def draft_clause(draft, table=None): + """get proper clause in build/rpm query for draft option :param bool draft: draft option: - True: append "draft IS True" - False: append "draft IS NOT True" - None: do nothing - :param list clauses: clauses list to construct query by QueryProcessor, which the draft clause - to append + True: "draft IS True" + False: "draft IS NOT True" + + :param str table: the table(alias) the draft belongs to + :return: the generated clause. + :rtype: str or None + :raises: GenericError if draft is None """ + if draft is None: + raise koji.GenericError('draft cannot be None') if not table: table = '' else: table += '.' - if draft is None: - return if draft: - clauses.append(f'{table}draft IS TRUE') + return f'{table}draft IS TRUE' else: # null is included - clauses.append(f'{table}draft IS NOT TRUE') + return f'{table}draft IS NOT TRUE' def _clean_draft_link(promoted_build): - """remove the symlink of old builddir to prmoted builddir""" + """remove the symlink of old builddir to promoted builddir""" draft_info = (promoted_build.get('extra') or {}).get('draft', {}) if not (draft_info and draft_info.get('promoted')): # skipped as it's not a promoted build. From f29e5e143e901d44f36ea8df56a19a04dd46c84c Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:38 +0000 Subject: [PATCH 16/70] parse target_release from draft release instead of extra.draft.target_release --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index c82756e..1a7f294 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -1364,10 +1364,7 @@ def handle_import(goptions, session, args): b_state = koji.BUILD_STATES[draft_build['state']] if b_state != 'COMPLETE': error("draft build %s is expected as COMPLETE, got %s" % (draft_build['nvr'], b_state)) - target_release = draft_build.get('extra', {}).get('draft', {}).get('target_release') - if not target_release: - error("Invalid draft build: %s," - " no draft.target_release found in extra" % draft_build['nvr']) + target_release = koji.parse_target_release(draft_build) draft_target_nvr = "%s-%s-%s" % ( draft_build['name'], draft_build['version'], target_release ) diff --git a/koji/__init__.py b/koji/__init__.py index 19f565a..a5a1084 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -296,8 +296,9 @@ PRIO_DEFAULT = 20 DEFAULT_REQUEST_TIMEOUT = 60 * 60 * 12 DEFAULT_AUTH_TIMEOUT = 60 -# draft release format +# draft release constants DRAFT_RELEASE_FORMAT = '{release}#draft_{id}' +DRAFT_RELEASE_PARSING_REGEX = r'^(?P.+)#draft_(?P[0-9]+)$' # BEGIN kojikamid dup # @@ -3928,6 +3929,52 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): return walker.walk() +def gen_draft_release(buildinfo): + """Generate draft_release based on input build information + + Currently, it's generated as {target_release}#draft_{build_id} + + :param buildinfo: buildinfo with target "release", used to generate draft_release + :type buildinfo: dict + :return: draft release + :rtype: str + :raises GenericError: if missing any required field + """ + for k in ('id', 'release'): + if not buildinfo.get(k): + raise GenericError("key: '%s' not found in %s" % (k, buildinfo)) + return DRAFT_RELEASE_FORMAT.format(**buildinfo) + + +def parse_target_release(buildinfo): + """Generate target_release from a draft buildinfo reversely + + :param buildinfo: buildinfo of a draft build + :type buildinfo: dict + :rtype: str + :raises GenericError: if missing draft field in buildinfo + :raises GenericError: if cannot get a valid target_release + """ + draft_release = buildinfo.get('release') + if not draft_release: + raise GenericError("'release' not found in %s" % buildinfo) + build_id = buildinfo.get('id') + if build_id: + build_id = str(build_id) + else: + raise GenericError("'id' not found in %s" % buildinfo) + match = re.match(DRAFT_RELEASE_PARSING_REGEX, draft_release) + if match: + if build_id != match.group('id'): + raise GenericError( + "buildinfo.id: %s doesn't match build id part: %s in buildinfo.release: %s" % + (build_id, match.group('id'), draft_release) + ) + return match.group('release') + else: + raise GenericError("draft release: %s is not in valid format" % draft_release) + + def add_file_logger(logger, fn): if not os.path.exists(fn): try: diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 0c2d440..7be5c61 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6104,7 +6104,7 @@ def new_build(data, strict=False): # handle draft suffix in release if data.get('draft'): target_release = data['release'] - data['release'] = insert_data['release'] = koji.DRAFT_RELEASE_FORMAT.format(**data) + data['release'] = insert_data['release'] = koji.gen_draft_release(data) # it's still possible to already have a build with the same nvr draft_nvr = dslice(data, ['name', 'version', 'release']) if find_build_id(draft_nvr): @@ -6413,13 +6413,8 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, d # only enforce the srpm name matching the build for non-wrapper rpms nvrinfo = buildinfo.copy() if draft: - # for draft build the release is buildinfo.extra.draft.target_release - target_release = (buildinfo.get('extra') or {}).get('draft', {}).get('target_release') - if not target_release: - raise koji.GenericError( - f'target release of draft build not found in extra of build: {buildinfo}' - ) - nvrinfo['release'] = target_release + # for draft build, change release to target_release + nvrinfo['release'] = koji.parse_target_release(buildinfo) srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % nvrinfo # either the sourcerpm field should match the build, or the filename # itself (for the srpm) @@ -15996,17 +15991,20 @@ def _promote_build(build, user=None, strict=True, force=False): else: return None old_release = binfo['release'] - extra = binfo.get('extra') or {} - # get target release in binfo.extra - draft_info = extra.get('draft', {}).copy() - target_release = draft_info.get('target_release') + + # get target release + target_release = None + try: + target_release = koji.parse_target_release(binfo) + except Exception: + if strict: + raise if not target_release: if strict: - raise koji.GenericError( - f'draft.target_release not found in extra of build: {binfo}' - ) - else: - return None + # should never happen + raise koji.GenericError('Cannot generate target_release') + return None + # drop id to get build by NVR target_build = dslice(binfo, ['name', 'version']) target_build['release'] = target_release @@ -16040,7 +16038,8 @@ def _promote_build(build, user=None, strict=True, force=False): # temp solution with python generated time, maybe better to be an event? now = datetime.datetime.now() - extra['draft'] = draft_info + extra = binfo.get('extra') or {} + draft_info = extra.setdefault('draft', {}) draft_info['promoted'] = True draft_info['old_release'] = old_release draft_info['promotion_time'] = encode_datetime(now) diff --git a/tests/test_cli/test_import.py b/tests/test_cli/test_import.py index 5fbb5f9..1b73569 100644 --- a/tests/test_cli/test_import.py +++ b/tests/test_cli/test_import.py @@ -1,9 +1,11 @@ from __future__ import absolute_import -import mock + import os -import six import unittest +import mock +import six + import koji from koji_cli.commands import handle_import from . import utils @@ -111,22 +113,24 @@ class TestImport(utils.CliTestCase): # check mock calls activate_session_mock.assert_called_with(session, options) - + get_header_fields_calls = [ mock.call(arguments[i], ('name', 'version', 'release', 'epoch', 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') ) for i in range(len(rpm_headers) - 1) - ] + ] get_header_fields_mock.assert_has_calls(get_header_fields_calls) - + if getrpm_calls: session.getRPM.assert_has_calls(getrpm_calls) elif getrpm_called: session.getRPM.assert_called_with( - dict((k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name']) + dict( + (k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name'] + ) ) else: session.getRPM.assert_not_called() @@ -174,8 +178,10 @@ class TestImport(utils.CliTestCase): ) if getrpm_called: session.getRPM.assert_called_with( - dict((k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name']) + dict( + (k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name'] + ) ) else: session.getRPM.assert_not_called() @@ -821,17 +827,13 @@ class TestImport(utils.CliTestCase): 'version': '4.4.12', 'release': '5.fc26#draft_123', 'nvr': 'bash-4.4.12-5.fc26', + 'id': '123', 'draft': True, - 'extra': { - 'draft': { - 'target_release': '5.fc26' - } - }, 'state': self.bstate['COMPLETE'] } session.getBuild.return_value = build - # Case 1. import src rpm with --draft-build + # Case 1. import src rpm with --draft-build # result: success expected = "uploading %s... done\n" % arguments[0] expected += "importing %s... done\n" % arguments[0] @@ -859,25 +861,21 @@ class TestImport(utils.CliTestCase): session = mock.MagicMock() session.getRPM.return_value = None build = { + 'id': '123', 'name': 'bash', 'version': '4.4.12', 'release': '5.fc26#draft_123', 'nvr': 'bash-4.4.12-5.fc26', 'draft': True, - 'extra': { - 'draft': { - 'target_release': '5.fc26' - } - }, 'state': self.bstate['COMPLETE'] } session.getBuild.return_value = build - # Case 1. import bin rpm with --draft-build + # Case 1. import bin rpm with --draft-build # result: success expected = "uploading %s... done\n" % arguments[0] expected += "importing %s... done\n" % arguments[0] - + self.__do_import_test( options, session, arguments, rpm_header=self.rpm_header, @@ -892,32 +890,55 @@ class TestImport(utils.CliTestCase): arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', '286'] options = mock.MagicMock() session = mock.MagicMock() - + cases = [ - # build, stderr + # build, stderr (None, "No such build: 286"), ({'draft': False, 'nvr': 'a-bad-draft'}, "a-bad-draft is not a draft build"), ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['DELETED']}, - "draft build a-bad-draft is expected as COMPLETE, got DELETED"), - ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], - 'extra': {'draft': {'no_target_release': 'omg'}}}, - "Invalid draft build: a-bad-draft, no draft.target_release found in extra") + "draft build a-bad-draft is expected as COMPLETE, got DELETED") ] for build, expected in cases: session.getBuild.return_value = build # result: error expected += "\n" self.assert_system_exit( - handle_import, - options, - session, - arguments, - stderr=expected, - activate_session=None, - exit_code=1) + handle_import, + options, + session, + arguments, + stderr=expected, + activate_session=None, + exit_code=1) options.reset_mock() session.reset_mock() + def test_handle_import_specified_draft_build_invalid_target_release(self): + """Test handle_import RPM import with --draft-build case. + Invalid target_release parsing + """ + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', '286'] + options = mock.MagicMock() + session = mock.MagicMock() + + cases = [ + # build, stderr + ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], + 'release': 'no_id'}, + "'id' not found in" + " {'draft': True, 'nvr': 'a-bad-draft', 'state': 1, 'release': 'no_id'}"), + ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], + 'id': 123, 'release': 'tgt#draft_234'}, + "buildinfo.id: 123 doesn't match build id part: 234" + " in buildinfo.release: tgt#draft_234") + ] + for build, expected in cases: + session.getBuild.return_value = build + # result: error + with self.assertRaises(koji.GenericError) as cm: + handle_import(options, session, arguments) + self.assertEqual(str(cm.exception), expected) + def test_handle_import_help(self): """Test handle_import function help message""" self.assert_help( diff --git a/tests/test_hub/test_import_rpm.py b/tests/test_hub/test_import_rpm.py index 25630e0..bd8acb0 100644 --- a/tests/test_hub/test_import_rpm.py +++ b/tests/test_hub/test_import_rpm.py @@ -78,7 +78,6 @@ class TestImportRPM(unittest.TestCase): kojihub.import_rpm(self.filename) self.assertEqual("Build is FAILED: name-version-release", str(cm.exception)) self.assertEqual(len(self.inserts), 0) - def test_import_rpm_completed_build(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' @@ -212,14 +211,13 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(insert.data, data) self.assertEqual(insert.rawdata, {}) - def test_import_draft_conflict(self): with self.assertRaises(koji.GenericError) as cm: kojihub.import_rpm(self.filename, buildinfo={'id': 1024, 'draft': False}, draft=True) self.assertEqual("draft property: False of build: 1024 mismatch, True is expected", str(cm.exception)) self.assertEqual(len(self.inserts), 0) - + def test_import_draft_rpm_without_buildinfo(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' self.get_rpm_header.return_value = self.rpm_header_retval @@ -229,8 +227,8 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(f"Cannot import draft rpm: {self.os_path_basename.return_value}" " without specifying a build", str(cm.exception)) self.assertEqual(len(self.inserts), 0) - - def test_import_draft_rpm_non_extra_target_release(self): + + def test_import_draft_rpm_invalid_release(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' self.get_rpm_header.return_value = self.rpm_header_retval @@ -238,7 +236,7 @@ class TestImportRPM(unittest.TestCase): 'state': koji.BUILD_STATES['DELETED'], 'name': 'name', 'version': 'version', - 'release': 'release', + 'release': 'badrelease', 'id': 12345, 'draft': True } @@ -246,11 +244,11 @@ class TestImportRPM(unittest.TestCase): with self.assertRaises(koji.GenericError) as cm: kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) self.assertEqual( - f'target release of draft build not found in extra of build: {buildinfo}', + 'draft release: badrelease is not in valid format', str(cm.exception) ) self.assertEqual(len(self.inserts), 0) - + def test_import_draft_rpm_valid(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' self.get_rpm_header.return_value = self.rpm_header_retval @@ -259,7 +257,7 @@ class TestImportRPM(unittest.TestCase): 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'name', 'version': 'version', - 'release': 'release', + 'release': 'release#draft_12345', 'id': 12345, 'draft': True, 'extra': { @@ -305,14 +303,9 @@ class TestImportRPM(unittest.TestCase): 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'name', 'version': 'version', - 'release': 'release', + 'release': 'release#draft_12345', 'id': 12345, - 'draft': True, - 'extra': { - 'draft': { - 'target_release': 'release' - } - } + 'draft': True } self.nextval.return_value = 9876 kojihub.import_rpm(self.src_filename, buildinfo=buildinfo, draft=True) @@ -384,4 +377,4 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(insert.data, data) self.assertEqual(insert.rawdata, {}) self.get_build.assert_called_once_with(5566, strict=True) - self.assertEqual(self.get_build.call_count, 1) \ No newline at end of file + self.assertEqual(self.get_build.call_count, 1) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index 7d7b87a..1653269 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -36,7 +36,9 @@ class TestPromoteBuild(unittest.TestCase): self.list_tags = mock.patch('kojihub.kojihub.list_tags', return_value=[{'id': 101}]).start() self.set_tag_update = mock.patch('kojihub.kojihub.set_tag_update').start() - self.encode_datetime = mock.patch('kojihub.kojihub.encode_datetime', return_value='NOW').start() + self.encode_datetime = mock.patch( + 'kojihub.kojihub.encode_datetime', return_value='NOW' + ).start() self._now = datetime.datetime.now() self._datetime = mock.patch('kojihub.kojihub.datetime.datetime').start() self.now = self._datetime.now = mock.MagicMock(return_value=self._now) @@ -45,12 +47,12 @@ class TestPromoteBuild(unittest.TestCase): 'id': 1, 'name': 'foo', 'version': 'bar', - 'release': 'dftrel_1', + 'release': 'tgtrel#draft_1', 'nvr': 'testnvr', 'extra': { 'draft': { 'promoted': False, - 'target_release': 'tgtrel_1' + 'target_release': 'tgtrel' }}, 'state': 1, 'draft': True, @@ -58,13 +60,13 @@ class TestPromoteBuild(unittest.TestCase): 'volume_name': 'X', 'task_id': 222 } - - self.new_build = { + + self.new_build = { # no check on the info 'id': 1, 'name': 'foo', 'version': 'bar', - 'release': 'tgtrel_1', + 'release': 'tgtrel', 'volume_name': 'X' } @@ -82,15 +84,15 @@ class TestPromoteBuild(unittest.TestCase): { 'draft': { 'promoted': True, - 'target_release': 'tgtrel_1', - 'old_release': 'dftrel_1', + 'target_release': 'tgtrel', + 'old_release': 'tgtrel#draft_1', 'promotion_time': 'NOW', 'promotion_ts': self._now.timestamp(), 'promoter': self.user['name'] } } ) - + ret = self.exports.promoteBuild('a-draft-build', strict=True) self.assertEqual(ret, self.new_build) self.assertEqual(len(self.updates), 1) @@ -98,7 +100,7 @@ class TestPromoteBuild(unittest.TestCase): self.assertEqual(update.table, 'build') self.assertEqual(update.values, self.draft_build) self.assertEqual(update.data, {'draft': False, - 'release': 'tgtrel_1', + 'release': 'tgtrel', 'extra': extra}) self.assertEqual(update.rawdata, {}) self.assertEqual(update.clauses, ['id=%(id)i']) @@ -113,21 +115,22 @@ class TestPromoteBuild(unittest.TestCase): self.exports.promoteBuild('a-regular-build', strict=True) self.assertEqual(str(cm.exception), "Not a draft build: {'draft': False}") self.assertEqual(len(self.updates), 0) - + ret = self.exports.promoteBuild('a-regular-build', strict=False) self.assertIsNone(ret) self.assertEqual(len(self.updates), 0) - def test_promote_build_no_target_release(self): + def test_promote_build__target_release(self): draft = { 'id': 1, 'name': 'foo', 'version': 'bar', - 'release': 'dftrel_1', + 'release': 'tgtrel#draft_2', 'extra': { 'draft': { - 'promoted': False - # no target_release + 'promoted': False, + # target_release doesn't matter now + 'target_release': 'any' }}, 'state': 1, 'draft': True, @@ -135,14 +138,17 @@ class TestPromoteBuild(unittest.TestCase): 'volume_name': 'X', 'task_id': 222 } - + self.get_build.return_value = draft with self.assertRaises(koji.GenericError) as cm: self.exports.promoteBuild('a-regular-build', strict=True) - self.assertEqual(str(cm.exception), f"draft.target_release not found in extra of build: {draft}") + self.assertEqual( + str(cm.exception), + "buildinfo.id: 1 doesn't match build id part: 2 in buildinfo.release: tgtrel#draft_2" + ) self.assertEqual(len(self.updates), 0) - + ret = self.exports.promoteBuild('a-regular-build', strict=False) self.assertIsNone(ret) self.assertEqual(len(self.updates), 0) @@ -152,12 +158,12 @@ class TestPromoteBuild(unittest.TestCase): 'id': 1, 'name': 'foo', 'version': 'bar', - 'release': 'dftrel_1', + 'release': 'tgtrel#draft_1', 'nvr': 'testnvr', 'extra': { 'draft': { - 'promoted': False - # no target_release + 'promoted': False, + 'target_release': 'any' }}, 'draft': True, 'state': 0, @@ -193,9 +199,9 @@ class TestPromoteBuild(unittest.TestCase): self.get_build.assert_called_with({ 'name': 'foo', 'version': 'bar', - 'release': 'tgtrel_1' + 'release': 'tgtrel' }) - + self.get_build.reset_mock() self.get_build.side_effect = [self.draft_build, old] ret = self.exports.promoteBuild('a-regular-build', strict=False) From 753f6911488d3196c32955058a9d3a37274ba4f6 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:39 +0000 Subject: [PATCH 17/70] cli: clean changes in handle_import --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 1a7f294..814b656 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -1328,10 +1328,6 @@ def handle_import(goptions, session, args): parser.add_option("--test", action="store_true", help="Don't actually import") parser.add_option("--create-build", action="store_true", help="Auto-create builds as needed") parser.add_option("--src-epoch", help="When auto-creating builds, use this epoch") - parser.add_option("--create-draft", action="store_true", - help="Auto-create draft builds instead as needed") - parser.add_option("--draft-build", metavar='NVR|ID', - help="The target draft build to import to") (options, args) = parser.parse_args(args) if len(args) < 1: parser.error("At least one package must be specified") @@ -1342,32 +1338,6 @@ def handle_import(goptions, session, args): options.src_epoch = int(options.src_epoch) except (ValueError, TypeError): parser.error("Invalid value for epoch: %s" % options.src_epoch) - if options.create_draft: - print("Will create draft build instead if desired nvr doesn't exist") - options.create_build = True - draft_build = None - draft_target_nvr = None - if options.draft_build: - if options.create_build: - parser.error( - "To ensure no misuse, don't specify --draft-build while auto-creating." - ) - try: - draft_build = int(options.draft_build) - except ValueError: - draft_build = options.draft_build - draft_build = session.getBuild(draft_build) - if not draft_build: - error("No such build: %s" % options.draft_build) - if not draft_build.get('draft'): - error("%s is not a draft build" % draft_build['nvr']) - b_state = koji.BUILD_STATES[draft_build['state']] - if b_state != 'COMPLETE': - error("draft build %s is expected as COMPLETE, got %s" % (draft_build['nvr'], b_state)) - target_release = koji.parse_target_release(draft_build) - draft_target_nvr = "%s-%s-%s" % ( - draft_build['name'], draft_build['version'], target_release - ) activate_session(session, goptions) to_import = {} for path in args: @@ -1379,7 +1349,6 @@ def handle_import(goptions, session, args): else: nvr = "%(name)s-%(version)s-%(release)s" % koji.parse_NVRA(data['sourcerpm']) to_import.setdefault(nvr, []).append((path, data)) - builds_missing = False nvrs = sorted(to_list(to_import.keys())) for nvr in nvrs: @@ -1388,41 +1357,29 @@ def handle_import(goptions, session, args): if data['sourcepackage']: break else: - if nvr == draft_target_nvr: - continue - # when no srpm and create_draft - elif options.create_draft: - print("Missing srpm for draft build creating with target nvr: %s" % nvr) - builds_missing = True - continue # no srpm included, check for build binfo = session.getBuild(nvr) if not binfo: print("Missing build or srpm: %s" % nvr) builds_missing = True - if builds_missing and (not options.create_build or options.create_draft): + if builds_missing and not options.create_build: print("Aborting import") return # local function to help us out below - def do_import(path, data, draft_build=None): - draft = bool(draft_build) or options.create_draft + def do_import(path, data): rinfo = dict([(k, data[k]) for k in ('name', 'version', 'release', 'arch')]) - if draft_build or not options.create_draft: - opts = {} - if draft_build: - opts['build'] = draft_build - prev = session.getRPM(rinfo, **opts) - if prev and not prev.get('external_repo_id', 0): - if prev['payloadhash'] == koji.hex_string(data['sigmd5']): - print("RPM already imported: %s" % path) - else: - warn("md5sum mismatch for %s" % path) - warn(" A different rpm with the same name has already been imported") - warn(" Existing sigmd5 is %r, your import has %r" % ( - prev['payloadhash'], koji.hex_string(data['sigmd5']))) - print("Skipping import") - return + prev = session.getRPM(rinfo) + if prev and not prev.get('external_repo_id', 0): + if prev['payloadhash'] == koji.hex_string(data['sigmd5']): + print("RPM already imported: %s" % path) + else: + warn("md5sum mismatch for %s" % path) + warn(" A different rpm with the same name has already been imported") + warn(" Existing sigmd5 is %r, your import has %r" % ( + prev['payloadhash'], koji.hex_string(data['sigmd5']))) + print("Skipping import") + return if options.test: print("Test mode -- skipping import for %s" % path) return @@ -1438,69 +1395,41 @@ def handle_import(goptions, session, args): sys.stdout.write("importing %s... " % path) sys.stdout.flush() try: - opts = {} - if draft_build: - opts['build'] = draft_build - if draft: - opts['draft'] = draft - rpminfo = session.importRPM(serverdir, os.path.basename(path), **opts) + session.importRPM(serverdir, os.path.basename(path)) except koji.GenericError as e: - rpminfo = None print("\nError importing: %s" % str(e).splitlines()[-1]) sys.stdout.flush() else: print("done") sys.stdout.flush() - return rpminfo for nvr in nvrs: # check for existing build need_build = True - is_draft = True - if nvr == draft_target_nvr: - binfo = draft_build - need_build = False - elif options.create_draft: - binfo = None - need_build = True - else: - is_draft = False - binfo = session.getBuild(nvr) - if binfo: - b_state = koji.BUILD_STATES[binfo['state']] - if b_state == 'COMPLETE': - need_build = False - elif b_state in ['FAILED', 'CANCELED']: - if not options.create_build: - print("Build %s state is %s. Skipping import" % (nvr, b_state)) - continue - else: - print("Build %s exists with state=%s. Skipping import" % (nvr, b_state)) + binfo = session.getBuild(nvr) + if binfo: + b_state = koji.BUILD_STATES[binfo['state']] + if b_state == 'COMPLETE': + need_build = False + elif b_state in ['FAILED', 'CANCELED']: + if not options.create_build: + print("Build %s state is %s. Skipping import" % (nvr, b_state)) continue + else: + print("Build %s exists with state=%s. Skipping import" % (nvr, b_state)) + continue # import srpms first, if any for path, data in to_import[nvr]: if data['sourcepackage']: - # we can not fix state for draft build by createEmptyBuild - if not is_draft and binfo and b_state != 'COMPLETE': + if binfo and b_state != 'COMPLETE': # need to fix the state print("Creating empty build: %s" % nvr) b_data = koji.util.dslice(binfo, ['name', 'version', 'release']) b_data['epoch'] = data['epoch'] session.createEmptyBuild(**b_data) binfo = session.getBuild(nvr) - dbld = binfo if is_draft else None - will_create = False - if options.create_draft and not dbld: - will_create = True - print("Will create draft build with target nvr: %s while importing" % nvr) - rpminfo = do_import(path, data, draft_build=dbld) - # only needed for draft build, but - # TODO: should be able to apply to regular import - if rpminfo and rpminfo.get('build', {}).get('draft'): - binfo = rpminfo['build'] - if will_create: - print("Draft build: %s created" % binfo['nvr']) + do_import(path, data) need_build = False if need_build: @@ -1509,11 +1438,11 @@ def handle_import(goptions, session, args): if binfo: # should have caught this earlier, but just in case... b_state = koji.BUILD_STATES[binfo['state']] - print("Build %s state is %s. Skipping import" % (binfo['nvr'], b_state)) + print("Build %s state is %s. Skipping import" % (nvr, b_state)) continue else: print("No such build: %s (include matching srpm or use " - "--create-build/--create-draft option to add it)" % nvr) + "--create-build option to add it)" % nvr) continue else: # let's make a new build @@ -1524,22 +1453,17 @@ def handle_import(goptions, session, args): # pull epoch from first rpm data = to_import[nvr][0][1] b_data['epoch'] = data['epoch'] - if options.create_draft: - b_data['draft'] = True - b_display = "empty draft build with target nvr: %s" % nvr - else: - b_display = "empty build: %s" % nvr if options.test: - print("Test mode -- would have created %s" % b_display) + print("Test mode -- would have created empty build: %s" % nvr) else: - print("Creating %s" % b_display) - buildid = session.createEmptyBuild(**b_data) - binfo = session.getBuild(buildid) + print("Creating empty build: %s" % nvr) + session.createEmptyBuild(**b_data) + binfo = session.getBuild(nvr) for path, data in to_import[nvr]: if data['sourcepackage']: continue - do_import(path, data, draft_build=binfo if is_draft else None) + do_import(path, data) def handle_import_cg(goptions, session, args): From 08a2d6c389d9c137bd08133c4c7818a9d1090e34 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:39 +0000 Subject: [PATCH 18/70] revert tests/test_cli/test_import.py --- diff --git a/tests/test_cli/test_import.py b/tests/test_cli/test_import.py index 1b73569..e91a5a7 100644 --- a/tests/test_cli/test_import.py +++ b/tests/test_cli/test_import.py @@ -1,10 +1,8 @@ from __future__ import absolute_import - -import os -import unittest - import mock +import os import six +import unittest import koji from koji_cli.commands import handle_import @@ -89,22 +87,15 @@ class TestImport(utils.CliTestCase): def __do_import_test(self, options, session, arguments, **kwargs): expected = kwargs.get('expected', None) rpm_header = kwargs.get('rpm_header', {}) - rpm_headers = kwargs.get('rpm_headers', []) - if not rpm_headers: - rpm_headers = [rpm_header] fake_srv_path = kwargs.get('srv_path', '/path/to/server/import') upload_rpm_mock = kwargs.get('upload_rpm_mock', session.uploadWrapper) - getrpm_called = kwargs.get('getrpm_called', True) - getrpm_calls = kwargs.get('getrpm_calls', []) - import_opts = kwargs.get('import_opts', {}) - import_rpm_calls = kwargs.get('import_rpm_calls', None) with mock.patch('koji.get_header_fields') as get_header_fields_mock: with mock.patch('koji_cli.commands.unique_path') as unique_path_mock: with mock.patch('koji_cli.commands.activate_session') as activate_session_mock: with mock.patch('sys.stdout', new_callable=six.StringIO) as stdout: with upload_rpm_mock: - get_header_fields_mock.side_effect = rpm_headers + get_header_fields_mock.return_value = rpm_header unique_path_mock.return_value = fake_srv_path handle_import(options, session, arguments) @@ -113,35 +104,21 @@ class TestImport(utils.CliTestCase): # check mock calls activate_session_mock.assert_called_with(session, options) + get_header_fields_mock.assert_called_with( + arguments[0], + ('name', 'version', 'release', 'epoch', + 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') + ) - get_header_fields_calls = [ - mock.call(arguments[i], - ('name', 'version', 'release', 'epoch', - 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') - ) for i in range(len(rpm_headers) - 1) - ] - - get_header_fields_mock.assert_has_calls(get_header_fields_calls) - - if getrpm_calls: - session.getRPM.assert_has_calls(getrpm_calls) - elif getrpm_called: - session.getRPM.assert_called_with( - dict( - (k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name'] - ) - ) - else: - session.getRPM.assert_not_called() + session.getRPM.assert_called_with( + dict((k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name']) + ) unique_path_mock.assert_called_with('cli-import') upload_rpm_mock.assert_called_with(arguments[0], self.fake_srv_dir) - if import_rpm_calls: - session.importRPM.assert_has_calls(import_rpm_calls) - else: - session.importRPM.assert_called_with( - self.fake_srv_dir, os.path.basename(arguments[0]), **import_opts) + session.importRPM.assert_called_with( + self.fake_srv_dir, os.path.basename(arguments[0])) # reset for next test activate_session_mock.reset_mock() @@ -159,7 +136,6 @@ class TestImport(utils.CliTestCase): expected = kwargs.get('expected', None) expected_warn = kwargs.get('expected_warn', None) rpm_header = kwargs.get('rpm_header', {}) - getrpm_called = kwargs.get('getrpm_called', True) with mock.patch('koji.get_header_fields') as get_header_fields_mock: get_header_fields_mock.return_value = rpm_header @@ -176,15 +152,11 @@ class TestImport(utils.CliTestCase): ('name', 'version', 'release', 'epoch', 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') ) - if getrpm_called: - session.getRPM.assert_called_with( - dict( - (k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name'] - ) - ) - else: - session.getRPM.assert_not_called() + + session.getRPM.assert_called_with( + dict((k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name']) + ) session.uploadWrapper.assert_not_called() session.importRPM.assert_not_called() @@ -705,240 +677,6 @@ class TestImport(utils.CliTestCase): activate_session=None) activate_session_mock.assert_not_called() - @mock.patch('sys.stdout', new_callable=six.StringIO) - @mock.patch('sys.stderr', new_callable=six.StringIO) - @mock.patch('koji_cli.commands.activate_session') - def test_handle_import_src_rpm_with_create_draft( - self, - activate_session_mock, - stderr, - stdout): - """Test handle_import SRPM import with creating draft build case.""" - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--create-draft'] - options = mock.MagicMock() - session = mock.MagicMock() - session.importRPM.return_value = {'build': {'nvr': 'a-draft-build', 'draft': True}} - - nvr = '%(name)s-%(version)s-%(release)s' % self.srpm_header - - # Case 1. import src rpm with --create-draft - # result: success - expected = "Will create draft build instead if desired nvr doesn't exist\n" - expected += "Will create draft build with target nvr: %s while importing\n" % nvr - expected += "uploading %s... done\n" % arguments[0] - expected += "importing %s... done\n" % arguments[0] - expected += "Draft build: a-draft-build created\n" - - self.__do_import_test( - options, session, arguments, - rpm_header=self.srpm_header, - expected=expected, - getrpm_called=False, - import_opts={'draft': True}) - - @mock.patch('sys.stdout', new_callable=six.StringIO) - @mock.patch('sys.stderr', new_callable=six.StringIO) - @mock.patch('koji_cli.commands.activate_session') - def test_handle_import_binary_rpm_with_create_draft( - self, - activate_session_mock, - stderr, - stdout): - """Test handle_import binary RPM import with creating draft build case.""" - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--create-draft'] - options = mock.MagicMock() - session = mock.MagicMock() - - nvr = '%(name)s-%(version)s-%(release)s' % self.rpm_header - - # Case 1. import bin rpm with --create-draft - # result: Aborting import - expected = "Will create draft build instead if desired nvr doesn't exist\n" - expected += "Missing srpm for draft build creating with target nvr: %s\n" % nvr - expected += "Aborting import\n" - self.__skip_import_test( - options, session, arguments, - rpm_header=self.rpm_header, - expected=expected, - getrpm_called=False) - - @mock.patch('sys.stdout', new_callable=six.StringIO) - @mock.patch('sys.stderr', new_callable=six.StringIO) - @mock.patch('koji_cli.commands.activate_session') - def test_handle_import_src_bin_rpm_with_create_draft( - self, - activate_session_mock, - stderr, - stdout): - """Test handle_import SRPM & RPM import with creating draft build case.""" - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '/path/to/bash-4.4.12-5.fc26.src.rpm', - '--create-draft'] - options = mock.MagicMock() - session = mock.MagicMock() - session.getRPM.return_value = None - session.importRPM.return_value = {'build': {'nvr': 'a-draft-build', 'draft': True}} - - nvr = '%(name)s-%(version)s-%(release)s' % self.srpm_header - - # Case 1. import src & bin rpm with --create-draft - # result: success - expected = "Will create draft build instead if desired nvr doesn't exist\n" - expected += "Will create draft build with target nvr: %s while importing\n" % nvr - expected += "uploading %s... done\n" % arguments[1] - expected += "importing %s... done\n" % arguments[1] - expected += "Draft build: a-draft-build created\n" - expected += "uploading %s... done\n" % arguments[0] - expected += "importing %s... done\n" % arguments[0] - - self.__do_import_test( - options, session, arguments, - rpm_headers=[self.rpm_header, self.srpm_header], - expected=expected, - getrpm_calls=[ - mock.call( - {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'x86_64'}, - build=session.importRPM.return_value['build'] - ) - ], - import_rpm_calls=[ - mock.call( - self.fake_srv_dir, os.path.basename(arguments[1]), draft=True - ), - mock.call( - self.fake_srv_dir, os.path.basename(arguments[0]), - build=session.importRPM.return_value['build'], draft=True - ) - ]) - - @mock.patch('sys.stdout', new_callable=six.StringIO) - @mock.patch('sys.stderr', new_callable=six.StringIO) - @mock.patch('koji_cli.commands.activate_session') - def test_handle_import_src_rpm_with_specified_draft_build( - self, - activate_session_mock, - stderr, - stdout): - """Test handle_import SRPM import with --draft-build case.""" - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', 'a-draft-build'] - options = mock.MagicMock() - session = mock.MagicMock() - build = { - 'name': 'bash', - 'version': '4.4.12', - 'release': '5.fc26#draft_123', - 'nvr': 'bash-4.4.12-5.fc26', - 'id': '123', - 'draft': True, - 'state': self.bstate['COMPLETE'] - } - session.getBuild.return_value = build - - # Case 1. import src rpm with --draft-build - # result: success - expected = "uploading %s... done\n" % arguments[0] - expected += "importing %s... done\n" % arguments[0] - - self.__do_import_test( - options, session, arguments, - rpm_header=self.srpm_header, - expected=expected, - getrpm_calls=[mock.call( - {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'src'}, - build=build)], - import_opts={'build': build, 'draft': True}) - - @mock.patch('sys.stdout', new_callable=six.StringIO) - @mock.patch('sys.stderr', new_callable=six.StringIO) - @mock.patch('koji_cli.commands.activate_session') - def test_handle_import_binary_rpm_with_specified_draft_build( - self, - activate_session_mock, - stderr, - stdout): - """Test handle_import RPM import with --draft-build case.""" - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', 'a-draft-build'] - options = mock.MagicMock() - session = mock.MagicMock() - session.getRPM.return_value = None - build = { - 'id': '123', - 'name': 'bash', - 'version': '4.4.12', - 'release': '5.fc26#draft_123', - 'nvr': 'bash-4.4.12-5.fc26', - 'draft': True, - 'state': self.bstate['COMPLETE'] - } - session.getBuild.return_value = build - - # Case 1. import bin rpm with --draft-build - # result: success - expected = "uploading %s... done\n" % arguments[0] - expected += "importing %s... done\n" % arguments[0] - - self.__do_import_test( - options, session, arguments, - rpm_header=self.rpm_header, - expected=expected, - getrpm_calls=[mock.call( - {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'x86_64'}, - build=build)], - import_opts={'build': build, 'draft': True}) - - def test_handle_import_specified_draft_build_invalid(self): - """Test handle_import RPM import with --draft-build case.""" - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', '286'] - options = mock.MagicMock() - session = mock.MagicMock() - - cases = [ - # build, stderr - (None, "No such build: 286"), - ({'draft': False, 'nvr': 'a-bad-draft'}, "a-bad-draft is not a draft build"), - ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['DELETED']}, - "draft build a-bad-draft is expected as COMPLETE, got DELETED") - ] - for build, expected in cases: - session.getBuild.return_value = build - # result: error - expected += "\n" - self.assert_system_exit( - handle_import, - options, - session, - arguments, - stderr=expected, - activate_session=None, - exit_code=1) - options.reset_mock() - session.reset_mock() - - def test_handle_import_specified_draft_build_invalid_target_release(self): - """Test handle_import RPM import with --draft-build case. - Invalid target_release parsing - """ - arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', '286'] - options = mock.MagicMock() - session = mock.MagicMock() - - cases = [ - # build, stderr - ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], - 'release': 'no_id'}, - "'id' not found in" - " {'draft': True, 'nvr': 'a-bad-draft', 'state': 1, 'release': 'no_id'}"), - ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], - 'id': 123, 'release': 'tgt#draft_234'}, - "buildinfo.id: 123 doesn't match build id part: 234" - " in buildinfo.release: tgt#draft_234") - ] - for build, expected in cases: - session.getBuild.return_value = build - # result: error - with self.assertRaises(koji.GenericError) as cm: - handle_import(options, session, arguments) - self.assertEqual(str(cm.exception), expected) - def test_handle_import_help(self): """Test handle_import function help message""" self.assert_help( @@ -953,8 +691,6 @@ Options: --create-build Auto-create builds as needed --src-epoch=SRC_EPOCH When auto-creating builds, use this epoch - --create-draft Auto-create draft builds instead as needed - --draft-build=NVR|ID The target draft build to import to """ % self.progname) From a3b6b8c43ab5f841bb281bc1fdc1a1a793bf3e2a Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:39 +0000 Subject: [PATCH 19/70] repoInit: reject duplicate nvras and write content.json in repodir --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 7be5c61..d24f143 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -2717,6 +2717,21 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non # Note: the repo_include_all option is not recommended for common use # see https://pagure.io/koji/issue/588 for background rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=True, latest=latest) + # we will iterate rpms twice, so covert it from generator to list + rpms = list(rpms) + # Since draft builds are introduced, we cannot guarantee the uniqueness of NVRA + # don't allow duplicate nvras in tag for repo generation. + rpm_mappings = {} + for r in rpms: + nvra = "{name}-{version}-{release}.{arch}".format(**r) + dup_rpms = rpm_mappings.setdefault(nvra, []) + dup_rpms.append(r['id']) + dups = {nvra: d for nvra, d in rpm_mappings.items() if len(d) > 1} + if dups: + raise koji.GenericError( + f"duplicate nvras found while repo#{repo_id} initiation: {dups}" + ) + groups = readTagGroups(tag_id, event=event_id, inherit=True) blocks = [pkg for pkg in readPackageList(tag_id, event=event_id, inherit=True, with_owners=False).values() @@ -2753,6 +2768,7 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non builddirs[build['id']] = relpath.lstrip('/') # generate pkglist files pkglist = {} + content_jsons = {} for repoarch in repo_arches: archdir = joinpath(repodir, repoarch) koji.ensuredir(archdir) @@ -2761,31 +2777,48 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non top_link = joinpath(archdir, 'toplink') os.symlink(top_relpath, top_link) pkglist[repoarch] = open(joinpath(archdir, 'pkglist'), 'wt', encoding='utf-8') + content_jsons[repoarch] = (joinpath(archdir, 'content.json'), {}) # NOTE - rpms is now an iterator for rpminfo in rpms: if not with_debuginfo and koji.is_debuginfo(rpminfo['name']): continue relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], relpathinfo.rpm(rpminfo)) + content = dslice( + rpminfo, + ( + 'id', 'build_id', 'name', 'version', 'release', 'epoch', + 'arch', 'payloadhash', 'size', 'buildtime' + ), + strict=True + ) + content['filepath'] = relpath + nvra = "{name}-{version}-{release}.{arch}".format(**content) arch = rpminfo['arch'] if arch == 'src': if with_src: for repoarch in repo_arches: pkglist[repoarch].write(relpath) + content_jsons[repoarch][1][nvra] = content if with_separate_src: pkglist[arch].write(relpath) + content_jsons[repoarch][1][nvra] = content elif arch == 'noarch': for repoarch in repo_arches: if repoarch == 'src': continue pkglist[repoarch].write(relpath) + content_jsons[repoarch][1][nvra] = content else: repoarch = koji.canonArch(arch) if repoarch not in repo_arches: # Do not create a repo for arches not in the arch list for this tag continue pkglist[repoarch].write(relpath) + content_jsons[repoarch][1][nvra] = content for repoarch in repo_arches: pkglist[repoarch].close() + with open(content_jsons[repoarch][0], 'wt', encoding='utf-8') as fp: + json.dump(content_jsons[repoarch][1], fp, indent=2) # write blocked package lists for repoarch in repo_arches: From a79c8ef468cae9f69ed63bb982021833b37b186a Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:40 +0000 Subject: [PATCH 20/70] kojid.BuildRoot: mapping id for each internal rpm by content.json --- diff --git a/builder/kojid b/builder/kojid index de7141e..7d5b6f1 100755 --- a/builder/kojid +++ b/builder/kojid @@ -204,6 +204,7 @@ class BuildRoot(object): self._load(*args, **kwargs) else: self._new(*args, **kwargs) + self._init_repo_data() def _load(self, data): # manage an existing buildroot @@ -346,6 +347,17 @@ class BuildRoot(object): with koji._open_text_file(configfile, 'wt') as fo: fo.write(output) + def _init_repo_data(self): + pathinfo = koji.PathInfo(topdir='') + self.repodir = pathinfo.repo(self.repoid, self.tag_name) + rel_repo_url = os.path.join(self.repodir, self.br_arch) + # repo_url can start with '/', don't use os.path.join + if self.options.topurl: + self.repo_url = '%s/%s' % (self.options.topurl, rel_repo_url) + elif self.options.topdir: + self.repo_url = '%s/%s' % (self.options.topdir, rel_repo_url) + logging.info("repo url of buildroot: %s is %s", self.name, self.repo_url) + def _repositoryEntries(self, pi, plugin=False): entries = [] if plugin: @@ -688,6 +700,7 @@ class BuildRoot(object): """Return a list of packages from the buildroot Each member of the list is a dictionary containing the following fields: + - id, optional for internal rpm available in content.json - name - version - release @@ -696,6 +709,8 @@ class BuildRoot(object): - payloadhash - size - buildtime + - external_repo, optional for external rpm + - location, optional for external rpm """ fields = ('name', 'version', @@ -734,6 +749,7 @@ class BuildRoot(object): finally: rpm.delMacro("_dbpath") self.markExternalRPMs(ret) + self.mapInternalRPMs(ret) return ret def getMavenPackageList(self, repodir): @@ -825,25 +841,16 @@ class BuildRoot(object): # substitute $arch in the url with the arch of the repo we're generating ext_url = erepo['url'].replace('$arch', self.br_arch) erepo_idx[ext_url] = erepo - pathinfo = koji.PathInfo(topdir='') - repodir = pathinfo.repo(self.repo_info['id'], self.repo_info['tag_name']) opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) opts['tempdir'] = self.options.workdir - repo_url = os.path.join(repodir, self.br_arch) - # repo_url can start with '/', don't use os.path.join - if self.options.topurl: - repo_url = '%s/%s' % (self.options.topurl, repo_url) - elif self.options.topdir: - repo_url = '%s/%s' % (self.options.topdir, repo_url) - logging.error(repo_url) tmpdir = os.path.join(self.tmpdir(), 'librepo-markExternalRPMs') koji.ensuredir(tmpdir) h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) - h.setopt(librepo.LRO_URLS, [repo_url]) + h.setopt(librepo.LRO_URLS, [self.repo_url]) h.setopt(librepo.LRO_DESTDIR, tmpdir) # We are using this just to find out location of 'origin', # we don't even need to download it since we use openRemoteFile @@ -852,7 +859,7 @@ class BuildRoot(object): pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href'] koji.util.rmtree(tmpdir) - relpath = os.path.join(repodir, self.br_arch, pkgorigins) + relpath = os.path.join(self.repodir, self.br_arch, pkgorigins) with koji.openRemoteFile(relpath, **opts) as fo: # at this point we know there were external repos at the create event, # so there should be an origins file. @@ -886,6 +893,61 @@ class BuildRoot(object): rpm_info['external_repo'] = erepo rpm_info['location'] = erepo['external_repo_id'] + def mapInternalRPMs(self, rpmlist): + """ + Map each rpm item of rpmlist by attaching rpm_id in repodir's content.json which + is generated while repo initiation. + + Skipped if content.json doesn't exist for backward compatibility. + + For now, content.json is a dict[nvra, rpmdata], where rpmdata contains keys below: + - id + - build_id + - name + - version + - release + - epoch + - arch + - payloadhash + - size + - buildtime + + Notes: + - Because the nvra uniqueness is still being kept, we don't need the check so far. + - it must be called after markExternalRPMs, as it use "external_repo" to skip external + rpms. + + :param list rpmlist: rpm list fetched from local RPMDB. + :return: the same rpmlist as the parameter with build nvrs attached + :rtype: list + """ + + opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) + content_json_file = os.path.join(self.repodir, self.br_arch, 'pkglist') + if not os.path.exists(content_json_file): + self.logger.warning( + "% not found, don't mapping internal rpms with ids for repo#%i", + content_json_file, self.repoid) + with koji.openRemoteFile(content_json_file, **opts) as fo: + content = json.load(fo) + + for rpm_info in rpmlist: + if 'external_repo' in rpm_info: + continue + nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info + data = content.get(nvra) + if not data: + self.logger.warning("%s not found in content.json", nvra) + continue + # check payloadhash in case they are different + elif data['payloadhash'] != rpm_info['payloadhash']: + self.logger.error( + "RPM: %s: payloadhash: %s mismatch expected %s in content.json", + nvra, rpm_info['payloadhash'], data['payloadhash']) + else: + # set rpm id + rpm_info['id'] = data['id'] + def path_without_to_within(self, path): """ Convert an absolute path from without the BuildRoot to one within. From 44209718d9794215c6e9b14fcc7b2f16cde11b99 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:40 +0000 Subject: [PATCH 21/70] reject draft build in buildroot of CG --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index d24f143..feedb59 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -7051,12 +7051,13 @@ class CG_Importer(object): # not in metadata spec, and will confuse get_rpm raise koji.GenericError("Unexpected 'id' field in component") # rpm is no more unique with NVRA as draft build is introduced - # TODO: we should consider how to handle them once draft build is enabled for CG rinfo = get_rpm(comp, strict=False) if not rinfo: # XXX - this is a temporary workaround until we can better track external refs self.log_warning("IGNORING unmatched rpm component: %r" % comp) return None + # TODO: we should consider how to handle them once draft build is enabled for CG + reject_draft(rinfo, is_rpm=True, strict=True) if rinfo['payloadhash'] != comp['sigmd5']: # XXX - this is a temporary workaround until we can better track external refs self.log_warning("IGNORING rpm component (md5 mismatch): %r" % comp) @@ -15946,19 +15947,24 @@ def create_rpm_checksum(rpm_id, sigkey, chsum_dict): insert.execute() -def reject_draft(buildinfo, error=None): - """block draft build +def reject_draft(data, is_rpm=False, error=None, strict=True): + """block draft build/rpm TODO: remove this once draft build is open for all build types - :param dict buildinfo: buildinfo dict + :param dict data: buildinfo dict or rpminfo dict if is_rpm is true + :param bool is_rpm: indicates data is rpm or build (true/false) in default error msg :param koji.GenericError error: the error raised if not a draft build, defaults to None to raise the default "unsupported" error - :raises error: default or specified by input error when draft==True in buldinfo + :param bool strict: if True will raise an error if it's draft, else return the draft value. + :raises error: default or specified by input error when draft==True in data """ - if buildinfo.get('draft'): + if not strict: + return bool(data.get('draft')) + if data.get('draft'): if error is None: - error = koji.GenericError("Draft build not supported") + entry_type = 'rpm' if is_rpm else 'build' + error = koji.GenericError(f"Draft {entry_type} not supported") raise error From 72fa1b8c580f2ff722c1c2f4310674a96683aff4 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:40 +0000 Subject: [PATCH 22/70] reject draft build in buildroot of winbuild --- diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 4f08b94..e35a39e 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -377,6 +377,11 @@ class WindowsBuild(object): brtype = brinfo.get('type', 'win') buildinfo = self.server.getLatestBuild(self.build_tag, buildreq, self.task_opts.get('repo_id')) + # don't allow draft build in buildrequires + if buildinfo.get('draft'): + raise BuildError( # noqa: F821 + "Draft build: %s is not supported in buildrequires" % buildinfo['nvr'] + ) br_dir = os.path.join(self.buildreq_dir, buildreq, brtype) ensuredir(br_dir) # noqa: F821 brinfo['dir'] = br_dir From c06e7f9ba0068e1d98da8ad0b2ba5ace1bb984d6 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:41 +0000 Subject: [PATCH 23/70] reject draft rpms as rpm components of an image build --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index feedb59..003c591 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -7678,7 +7678,9 @@ def new_image_build(build_info): def new_typed_build(build_info, btype): """Mark build as a given btype""" - + # add here in case disabling draft build for non-rpm is missing before calling this + if btype != 'rpm': + reject_draft(build_info) btype_id = lookup_name('btype', btype, strict=True)['id'] query = QueryProcessor(tables=['build_types'], columns=['build_id'], clauses=['build_id = %(build_id)i', 'btype_id = %(btype_id)i'], @@ -10210,6 +10212,9 @@ def importImageInternal(task_id, build_info, imgdata): data = add_external_rpm(an_rpm, location, strict=False) else: data = get_rpm(an_rpm, strict=True) + # unlike buildroot, we simply reject draft rpms as rpm components + # because we probably don't want to keep the nvra uniqueness here. + reject_draft(data, is_rpm=True) rpm_ids.append(data['id']) # we sort to try to avoid deadlock issues rpm_ids.sort() @@ -14935,6 +14940,7 @@ class HostExports(object): # ensure release is None so get_next_build will handle incrementing build_info['release'] = None data = build_info.copy() + reject_draft(data) data['task_id'] = task_id data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] @@ -15100,6 +15106,7 @@ class HostExports(object): task = Task(task_id) task.assertHost(host.id) data = build_info.copy() + reject_draft(data) data['task_id'] = task_id data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] @@ -15122,6 +15129,7 @@ class HostExports(object): task.assertHost(host.id) # build_info must contain name, version, and release data = build_info.copy() + reject_draft(data) data['task_id'] = task_id data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] @@ -15298,6 +15306,7 @@ class HostExports(object): importImageInternal(task_id, build_info, sub_results) if 'rpmresults' in sub_results: rpm_results = sub_results['rpmresults'] + # draft will be rejected _import_wrapper(rpm_results['task_id'], get_build(build_info['id'], strict=True), rpm_results) From 3b1df037be675dfd58e2385065e0f5cd576696a1 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:41 +0000 Subject: [PATCH 24/70] distRepo: denying draft builds --- diff --git a/builder/kojid b/builder/kojid index 7d5b6f1..a72102b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -6409,10 +6409,16 @@ enabled=1 for a in self.compat[arch]: # note: self.compat includes noarch for non-src already rpm_iter, builds = self.session.listTaggedRPMS(tag_id, - event=opts['event'], arch=a, + event=opts['event'], + arch=a, latest=opts['latest'], - inherit=opts['inherit'], rpmsigs=True) + inherit=opts['inherit'], + rpmsigs=True) + for build in builds: + # disable draft for distRepo so far + if build.get('draft'): + raise koji.BuildError("Draft build: %s is not allowed" % build['nvr']) builddirs[build['id']] = koji.pathinfo.build(build) rpms += list(rpm_iter) From 009de1e68822cf57f6365ad1df6da06f50aa800b Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jan 05 2024 10:00:42 +0000 Subject: [PATCH 25/70] fix typo --- diff --git a/builder/kojid b/builder/kojid index a72102b..a753a94 100755 --- a/builder/kojid +++ b/builder/kojid @@ -923,7 +923,7 @@ class BuildRoot(object): """ opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) - content_json_file = os.path.join(self.repodir, self.br_arch, 'pkglist') + content_json_file = os.path.join(self.repodir, self.br_arch, 'content.json') if not os.path.exists(content_json_file): self.logger.warning( "% not found, don't mapping internal rpms with ids for repo#%i", From ecdeba8c723c99e185d7cf8dfe0282767907ade4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:42 +0000 Subject: [PATCH 26/70] raise http errors in downloadFile --- diff --git a/koji/__init__.py b/koji/__init__.py index a5a1084..6799b36 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2028,6 +2028,7 @@ def downloadFile(url, path=None, fo=None): fo.write(chunk) finally: resp.close() + resp.raise_for_status() if resp.headers.get('Content-Length') and fo.tell() != int(resp.headers['Content-Length']): raise GenericError("Downloaded file %s doesn't match expected size (%s vs %s)" % (url, fo.tell(), resp.headers['Content-Length'])) From 8b9a44954cd01cfc415c50dbf5adfb2492c35864 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:42 +0000 Subject: [PATCH 27/70] temporary fallback code for missing content.json --- diff --git a/builder/kojid b/builder/kojid index a753a94..f9c4a89 100755 --- a/builder/kojid +++ b/builder/kojid @@ -924,13 +924,16 @@ class BuildRoot(object): opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) content_json_file = os.path.join(self.repodir, self.br_arch, 'content.json') - if not os.path.exists(content_json_file): - self.logger.warning( - "% not found, don't mapping internal rpms with ids for repo#%i", - content_json_file, self.repoid) - with koji.openRemoteFile(content_json_file, **opts) as fo: - content = json.load(fo) - + try: + with koji.openRemoteFile(content_json_file, **opts) as fo: + content = json.load(fo) + except requests.exceptions.HTTPError as e: + if '404' in str(e): + self.logger.warning("Missing repo content file: %s", content_json_file) + # TODO: remove this workaround once we can assume that repos contain this file + self.compat_map_internal_rpms(rpmlist) + return + raise for rpm_info in rpmlist: if 'external_repo' in rpm_info: continue @@ -948,6 +951,30 @@ class BuildRoot(object): # set rpm id rpm_info['id'] = data['id'] + def compat_map_internal_rpms(self, rpmlist): + drafts, draftbuilds = self.session.listTaggedRPMS( + tag=self.repo_info['tag_id'], + event=self.repo_info['create_event'], + latest=True, + draft=True) + fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" + content = {} + for draftinfo in drafts: + nvra = fmt % draftinfo + content[nvra] = draftinfo + for rpminfo in rpmlist: + nvra = fmt % rpminfo + data = content.get(nvra) + if data: + # draft rpm! + if data['payloadhash'] != rpminfo['payloadhash']: + self.logger.error( + "RPM: %s: payloadhash: %s mismatch expected %s", + nvra, rpminfo['payloadhash'], data['payloadhash']) + else: + # set rpm id + rpminfo['id'] = data['id'] + def path_without_to_within(self, path): """ Convert an absolute path from without the BuildRoot to one within. From ea86bd90b7adb97f32968edd216797494380cbb1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:42 +0000 Subject: [PATCH 28/70] change draft suffix format to be url friendly --- diff --git a/koji/__init__.py b/koji/__init__.py index 6799b36..4508b79 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -297,8 +297,8 @@ DEFAULT_REQUEST_TIMEOUT = 60 * 60 * 12 DEFAULT_AUTH_TIMEOUT = 60 # draft release constants -DRAFT_RELEASE_FORMAT = '{release}#draft_{id}' -DRAFT_RELEASE_PARSING_REGEX = r'^(?P.+)#draft_(?P[0-9]+)$' +DRAFT_RELEASE_FORMAT = '{release},draft_{id}' +DRAFT_RELEASE_PARSING_REGEX = r'^(?P.+),draft_(?P[0-9]+)$' # BEGIN kojikamid dup # @@ -3933,7 +3933,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): def gen_draft_release(buildinfo): """Generate draft_release based on input build information - Currently, it's generated as {target_release}#draft_{build_id} + Currently, it's generated as {target_release},draft_{build_id} :param buildinfo: buildinfo with target "release", used to generate draft_release :type buildinfo: dict From 71898e43f406bc099c92132a5209ce8b9898f15d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:43 +0000 Subject: [PATCH 29/70] show basic draft info in web ui --- diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index d905007..5b1fbe0 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -24,6 +24,11 @@ Epoch$build.epoch + #if $build.draft + DraftTrue + #else + DraftFalse + #end if #if $build.get('source') Source$build['source'] diff --git a/www/kojiweb/rpminfo.chtml b/www/kojiweb/rpminfo.chtml index 79e822a..40a3eb0 100644 --- a/www/kojiweb/rpminfo.chtml +++ b/www/kojiweb/rpminfo.chtml @@ -14,6 +14,11 @@ ID$rpm.id + #if $build + + Build$koji.buildLabel($build) + + #end if #if $build Name$util.escapeHTML($rpm.name) @@ -37,6 +42,12 @@ Arch$util.escapeHTML($rpm.arch) + + #if $rpm.draft + DraftTrue + #else + DraftFalse + #end if #if $rpm.external_repo_id == 0 Summary$util.escapeHTML($summary) From fc08fc0f743659c6ccf4e6f9e3ba3f7bc3dd58a5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:43 +0000 Subject: [PATCH 30/70] use nl-delimited json to dump rpm data --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 003c591..1ad6ffd 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -2717,20 +2717,6 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non # Note: the repo_include_all option is not recommended for common use # see https://pagure.io/koji/issue/588 for background rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=True, latest=latest) - # we will iterate rpms twice, so covert it from generator to list - rpms = list(rpms) - # Since draft builds are introduced, we cannot guarantee the uniqueness of NVRA - # don't allow duplicate nvras in tag for repo generation. - rpm_mappings = {} - for r in rpms: - nvra = "{name}-{version}-{release}.{arch}".format(**r) - dup_rpms = rpm_mappings.setdefault(nvra, []) - dup_rpms.append(r['id']) - dups = {nvra: d for nvra, d in rpm_mappings.items() if len(d) > 1} - if dups: - raise koji.GenericError( - f"duplicate nvras found while repo#{repo_id} initiation: {dups}" - ) groups = readTagGroups(tag_id, event=event_id, inherit=True) blocks = [pkg for pkg in readPackageList(tag_id, event=event_id, inherit=True, @@ -2766,9 +2752,9 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non for build in builds: relpath = relpathinfo.build(build) builddirs[build['id']] = relpath.lstrip('/') - # generate pkglist files + # generate pkglist and rpmlist files pkglist = {} - content_jsons = {} + rpmlist = {} for repoarch in repo_arches: archdir = joinpath(repodir, repoarch) koji.ensuredir(archdir) @@ -2777,48 +2763,43 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non top_link = joinpath(archdir, 'toplink') os.symlink(top_relpath, top_link) pkglist[repoarch] = open(joinpath(archdir, 'pkglist'), 'wt', encoding='utf-8') - content_jsons[repoarch] = (joinpath(archdir, 'content.json'), {}) - # NOTE - rpms is now an iterator + rpmlist[repoarch] = open(joinpath(archdir, 'rpmlist.jsonl'), 'wt', encoding='utf-8') + # NOTE - rpms is a generator for rpminfo in rpms: if not with_debuginfo and koji.is_debuginfo(rpminfo['name']): continue relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], relpathinfo.rpm(rpminfo)) - content = dslice( - rpminfo, - ( - 'id', 'build_id', 'name', 'version', 'release', 'epoch', - 'arch', 'payloadhash', 'size', 'buildtime' - ), - strict=True - ) - content['filepath'] = relpath - nvra = "{name}-{version}-{release}.{arch}".format(**content) + rpm_json = json.dumps(rpminfo, indent=None) + # must be one line for nl-delimited json arch = rpminfo['arch'] if arch == 'src': if with_src: for repoarch in repo_arches: pkglist[repoarch].write(relpath) - content_jsons[repoarch][1][nvra] = content + rpmlist[repoarch].write(rpm_json) + rpmlist[repoarch].write('\n') if with_separate_src: pkglist[arch].write(relpath) - content_jsons[repoarch][1][nvra] = content + rpmlist[arch].write(rpm_json) + rpmlist[arch].write('\n') elif arch == 'noarch': for repoarch in repo_arches: if repoarch == 'src': continue pkglist[repoarch].write(relpath) - content_jsons[repoarch][1][nvra] = content + rpmlist[repoarch].write(rpm_json) + rpmlist[repoarch].write('\n') else: repoarch = koji.canonArch(arch) if repoarch not in repo_arches: # Do not create a repo for arches not in the arch list for this tag continue pkglist[repoarch].write(relpath) - content_jsons[repoarch][1][nvra] = content + rpmlist[repoarch].write(rpm_json) + rpmlist[repoarch].write('\n') for repoarch in repo_arches: pkglist[repoarch].close() - with open(content_jsons[repoarch][0], 'wt', encoding='utf-8') as fp: - json.dump(content_jsons[repoarch][1], fp, indent=2) + rpmlist[repoarch].close() # write blocked package lists for repoarch in repo_arches: From 69d6f4139eb413f385a56ed664e5942f31f595ab Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:43 +0000 Subject: [PATCH 31/70] parse rpmlist.jsonl on builder --- diff --git a/builder/kojid b/builder/kojid index f9c4a89..c158c26 100755 --- a/builder/kojid +++ b/builder/kojid @@ -700,7 +700,7 @@ class BuildRoot(object): """Return a list of packages from the buildroot Each member of the list is a dictionary containing the following fields: - - id, optional for internal rpm available in content.json + - id, optional for internal rpm available in rpmlist.jsonl - name - version - release @@ -895,12 +895,12 @@ class BuildRoot(object): def mapInternalRPMs(self, rpmlist): """ - Map each rpm item of rpmlist by attaching rpm_id in repodir's content.json which + Map each rpm item of rpmlist by attaching rpm_id in repodir's rpmlist.jsonl which is generated while repo initiation. - Skipped if content.json doesn't exist for backward compatibility. + Skipped if rpmlist.jsonl doesn't exist for backward compatibility. - For now, content.json is a dict[nvra, rpmdata], where rpmdata contains keys below: + For now, rpmlist.jsonl is a dict[nvra, rpmdata], where rpmdata contains keys below: - id - build_id - name @@ -923,13 +923,17 @@ class BuildRoot(object): """ opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) - content_json_file = os.path.join(self.repodir, self.br_arch, 'content.json') + rpmlist_path = os.path.join(self.repodir, self.br_arch, 'rpmlist.jsonl') + repo_rpms = {} try: - with koji.openRemoteFile(content_json_file, **opts) as fo: - content = json.load(fo) + with koji.openRemoteFile(rpmlist_path, **opts) as fo: + for line in fo: + repo_rpm = json.loads(line) + nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % repo_rpm + repo_rpms[nvra] = repo_rpm except requests.exceptions.HTTPError as e: if '404' in str(e): - self.logger.warning("Missing repo content file: %s", content_json_file) + self.logger.warning("Missing repo content file: %s", rpmlist_path) # TODO: remove this workaround once we can assume that repos contain this file self.compat_map_internal_rpms(rpmlist) return @@ -938,14 +942,14 @@ class BuildRoot(object): if 'external_repo' in rpm_info: continue nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info - data = content.get(nvra) + data = repo_rpms.get(nvra) if not data: - self.logger.warning("%s not found in content.json", nvra) + self.logger.warning("%s not found in rpmlist.jsonl", nvra) continue # check payloadhash in case they are different elif data['payloadhash'] != rpm_info['payloadhash']: self.logger.error( - "RPM: %s: payloadhash: %s mismatch expected %s in content.json", + "RPM: %s: payloadhash: %s mismatch expected %s in rpmlist.jsonl", nvra, rpm_info['payloadhash'], data['payloadhash']) else: # set rpm id From 134360df2e0da3c8c5987ba1c42c2fd3ef9c8231 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:44 +0000 Subject: [PATCH 32/70] mark draft rpms in web ui lists --- diff --git a/www/kojiweb/rpmlist.chtml b/www/kojiweb/rpmlist.chtml index 6ba83aa..0586275 100644 --- a/www/kojiweb/rpmlist.chtml +++ b/www/kojiweb/rpmlist.chtml @@ -65,7 +65,7 @@ colspan="2" #slurp #for $rpm in $rpms #set $epoch = ($rpm.epoch != None and $str($rpm.epoch) + ':' or '') - $util.escapeHTML($rpm.name)-$epoch$rpm.version-$rpm.release.${rpm.arch}.rpm + $util.formatRPM($rpm) #if $type in ['component', 'image'] #if $rpm.external_repo_id == 0 internal diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index 5720db8..014add4 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -568,6 +568,25 @@ def formatLink(url): return url +def formatRPM(rpminfo, link=True): + """Format an rpm dict for display""" + rpminfo = rpminfo.copy() + if rpminfo.get('epoch'): + rpminfo['epoch'] = str(rpminfo['epoch']) + ':' + else: + rpminfo['epoch'] = '' + if rpminfo.get('draft'): + rpminfo['suffix'] = f" (draft_{rpminfo.get('build_id', '???')})" + else: + rpminfo['suffix'] = '' + label = escapeHTML("%(name)s-%(epoch)s%(version)s-%(release)s.%(arch)s%(suffix)s" % rpminfo) + if link: + rpm_id = rpminfo['id'] + return f'{label}' + else: + return label + + def rowToggle(template): """If the value of template._rowNum is even, return 'row-even'; if it is odd, return 'row-odd'. Increment the value before checking it. From 931bdb9292723752aef80dd3941a36f87f638aa6 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:44 +0000 Subject: [PATCH 33/70] explicitly check 404 status_code --- diff --git a/builder/kojid b/builder/kojid index c158c26..bdcca7d 100755 --- a/builder/kojid +++ b/builder/kojid @@ -932,7 +932,7 @@ class BuildRoot(object): nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % repo_rpm repo_rpms[nvra] = repo_rpm except requests.exceptions.HTTPError as e: - if '404' in str(e): + if e.response.status_code == 404: self.logger.warning("Missing repo content file: %s", rpmlist_path) # TODO: remove this workaround once we can assume that repos contain this file self.compat_map_internal_rpms(rpmlist) From 025b0a136f65168b5bda7f493e857ee2e1764ac1 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:44 +0000 Subject: [PATCH 34/70] error if payloadhash mismatch --- diff --git a/builder/kojid b/builder/kojid index bdcca7d..c171735 100755 --- a/builder/kojid +++ b/builder/kojid @@ -938,19 +938,21 @@ class BuildRoot(object): self.compat_map_internal_rpms(rpmlist) return raise + fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" for rpm_info in rpmlist: if 'external_repo' in rpm_info: continue - nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info + nvra = fmt % rpm_info data = repo_rpms.get(nvra) if not data: self.logger.warning("%s not found in rpmlist.jsonl", nvra) continue # check payloadhash in case they are different elif data['payloadhash'] != rpm_info['payloadhash']: - self.logger.error( - "RPM: %s: payloadhash: %s mismatch expected %s in rpmlist.jsonl", - nvra, rpm_info['payloadhash'], data['payloadhash']) + raise koji.BuildrootError( + "RPM: %s: payloadhash: %s mismatch expected %s in rpmlist.jsonl" + % (nvra, rpm_info['payloadhash'], data['payloadhash']) + ) else: # set rpm id rpm_info['id'] = data['id'] @@ -972,9 +974,10 @@ class BuildRoot(object): if data: # draft rpm! if data['payloadhash'] != rpminfo['payloadhash']: - self.logger.error( - "RPM: %s: payloadhash: %s mismatch expected %s", - nvra, rpminfo['payloadhash'], data['payloadhash']) + raise koji.BuildrootError( + "RPM: %s: payloadhash: %s mismatch expected %s" + % (nvra, rpminfo['payloadhash'], data['payloadhash']) + ) else: # set rpm id rpminfo['id'] = data['id'] From b4623d52ca95d958090c37fb23310d7916623e40 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:45 +0000 Subject: [PATCH 35/70] extra changes by draft suffix change --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 814b656..4050fcc 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -2822,7 +2822,7 @@ def anon_handle_list_tagged(goptions, session, args): else: fmt = "%(name)s-%(version)s-%(release)s.%(arch)s%(draft_suffix)s" for x in data: - x['draft_suffix'] = (' (#draft_%s)' % x['build_id']) if x.get('draft') else '' + x['draft_suffix'] = (' (,draft_%s)' % x['build_id']) if x.get('draft') else '' if options.sigs: fmt = "%(sigkey)s " + fmt else: @@ -2888,7 +2888,7 @@ def anon_handle_list_buildroot(goptions, session, args): line = nvra if options.verbose: if rinfo.get('draft'): - line += " (#draft_%s)" % rinfo['build_id'] + line += " (,draft_%s)" % rinfo['build_id'] if rinfo.get('is_update'): line += " [update]" print(line) diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 1ad6ffd..b66e820 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6050,7 +6050,7 @@ def new_build(data, strict=False): If the build to create is a draft, the release field is the target release rather than its actual release with draft suffix. The draft suffix will be - generated here as #draft_. + generated here as ",draft_". If strict is specified, raise an exception, if build already exists. """ From e3b43ba15ce296971cb70c79488862c85ed8861c Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:45 +0000 Subject: [PATCH 36/70] flake8 fixes --- diff --git a/builder/kojid b/builder/kojid index c171735..0c655db 100755 --- a/builder/kojid +++ b/builder/kojid @@ -959,10 +959,10 @@ class BuildRoot(object): def compat_map_internal_rpms(self, rpmlist): drafts, draftbuilds = self.session.listTaggedRPMS( - tag=self.repo_info['tag_id'], - event=self.repo_info['create_event'], - latest=True, - draft=True) + tag=self.repo_info['tag_id'], + event=self.repo_info['create_event'], + latest=True, + draft=True) fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" content = {} for draftinfo in drafts: @@ -6448,7 +6448,7 @@ enabled=1 latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) - + for build in builds: # disable draft for distRepo so far if build.get('draft'): From 5571627667a6c6e0e6e573c26a0dfa0342b2e11a Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:46 +0000 Subject: [PATCH 37/70] rework _init_repo_data to get_repo_dir --- diff --git a/builder/kojid b/builder/kojid index 0c655db..1eef7bd 100755 --- a/builder/kojid +++ b/builder/kojid @@ -204,7 +204,6 @@ class BuildRoot(object): self._load(*args, **kwargs) else: self._new(*args, **kwargs) - self._init_repo_data() def _load(self, data): # manage an existing buildroot @@ -347,16 +346,9 @@ class BuildRoot(object): with koji._open_text_file(configfile, 'wt') as fo: fo.write(output) - def _init_repo_data(self): + def get_repo_dir(self): pathinfo = koji.PathInfo(topdir='') - self.repodir = pathinfo.repo(self.repoid, self.tag_name) - rel_repo_url = os.path.join(self.repodir, self.br_arch) - # repo_url can start with '/', don't use os.path.join - if self.options.topurl: - self.repo_url = '%s/%s' % (self.options.topurl, rel_repo_url) - elif self.options.topdir: - self.repo_url = '%s/%s' % (self.options.topdir, rel_repo_url) - logging.info("repo url of buildroot: %s is %s", self.name, self.repo_url) + return pathinfo.repo(self.repoid, self.tag_name) def _repositoryEntries(self, pi, plugin=False): entries = [] @@ -845,12 +837,20 @@ class BuildRoot(object): opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) opts['tempdir'] = self.options.workdir + repo_url = os.path.join(self.get_repo_dir(), self.br_arch) + # repo_url can start with '/', don't use os.path.join + if self.options.topurl: + repo_url = '%s/%s' % (self.options.topurl, repo_url) + elif self.options.topdir: + repo_url = '%s/%s' % (self.options.topdir, repo_url) + self.logger.info("repo url of buildroot: %s is %s", self.name, repo_url) + tmpdir = os.path.join(self.tmpdir(), 'librepo-markExternalRPMs') koji.ensuredir(tmpdir) h = librepo.Handle() r = librepo.Result() h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) - h.setopt(librepo.LRO_URLS, [self.repo_url]) + h.setopt(librepo.LRO_URLS, [repo_url]) h.setopt(librepo.LRO_DESTDIR, tmpdir) # We are using this just to find out location of 'origin', # we don't even need to download it since we use openRemoteFile @@ -859,7 +859,7 @@ class BuildRoot(object): pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href'] koji.util.rmtree(tmpdir) - relpath = os.path.join(self.repodir, self.br_arch, pkgorigins) + relpath = os.path.join(self.get_repo_dir(), self.br_arch, pkgorigins) with koji.openRemoteFile(relpath, **opts) as fo: # at this point we know there were external repos at the create event, # so there should be an origins file. @@ -923,7 +923,7 @@ class BuildRoot(object): """ opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) - rpmlist_path = os.path.join(self.repodir, self.br_arch, 'rpmlist.jsonl') + rpmlist_path = os.path.join(self.get_repo_dir(), self.br_arch, 'rpmlist.jsonl') repo_rpms = {} try: with koji.openRemoteFile(rpmlist_path, **opts) as fo: From cc8ccbf45e79104458e14555e1f35c6368aa89cf Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:46 +0000 Subject: [PATCH 38/70] update draft_release_sane CONSTRAINT for draft suffix change --- diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index 804134a..19642a2 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -57,7 +57,7 @@ BEGIN; -- required by constraint rpminfo_build_id_draft_fkey on table rpminfo ALTER TABLE build ADD CONSTRAINT draft_for_rpminfo UNIQUE (id, draft); ALTER TABLE build ADD CONSTRAINT draft_release_sane CHECK - ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) + ((draft AND release ~ ('^.*,draft_' || id::TEXT || '$')) OR NOT draft); ALTER TABLE rpminfo ADD COLUMN draft BOOLEAN; diff --git a/schemas/schema.sql b/schemas/schema.sql index 5d820a8..f024122 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -300,7 +300,7 @@ CREATE TABLE build ( CONSTRAINT draft_for_rpminfo UNIQUE (id, draft), CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR (state <> 0 AND completion_time IS NOT NULL)), - CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) OR + CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*,draft_' || id::TEXT || '$')) OR NOT draft) ) WITHOUT OIDS; From 34c6712fdeeeac0a7d062caae1fcc02755da047c Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:47 +0000 Subject: [PATCH 39/70] simpler gen_draft_release and parse_target_release --- diff --git a/koji/__init__.py b/koji/__init__.py index 4508b79..113dd85 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -297,8 +297,8 @@ DEFAULT_REQUEST_TIMEOUT = 60 * 60 * 12 DEFAULT_AUTH_TIMEOUT = 60 # draft release constants -DRAFT_RELEASE_FORMAT = '{release},draft_{id}' -DRAFT_RELEASE_PARSING_REGEX = r'^(?P.+),draft_(?P[0-9]+)$' +DRAFT_RELEASE_DELIMITER = ',' +DRAFT_RELEASE_FORMAT = '{release}' + DRAFT_RELEASE_DELIMITER + 'draft_{id}' # BEGIN kojikamid dup # @@ -3930,50 +3930,33 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): return walker.walk() -def gen_draft_release(buildinfo): +def gen_draft_release(release, id): """Generate draft_release based on input build information Currently, it's generated as {target_release},draft_{build_id} - :param buildinfo: buildinfo with target "release", used to generate draft_release - :type buildinfo: dict + :param str release: target "release", which is the release part of rpms' nvra, + and will be the release of the build the draft build is going to be + promoted to. + :param int id: the build "id" part in draft_release (it's unchanged so it can be used to + keep the uniqueness of build NVR) :return: draft release :rtype: str - :raises GenericError: if missing any required field """ - for k in ('id', 'release'): - if not buildinfo.get(k): - raise GenericError("key: '%s' not found in %s" % (k, buildinfo)) - return DRAFT_RELEASE_FORMAT.format(**buildinfo) + return DRAFT_RELEASE_FORMAT.format(**locals()) -def parse_target_release(buildinfo): - """Generate target_release from a draft buildinfo reversely +def parse_target_release(draft_release): + """Generate target_release from a draft release reversely - :param buildinfo: buildinfo of a draft build - :type buildinfo: dict + :param str draft_release: release of a draft build :rtype: str - :raises GenericError: if missing draft field in buildinfo :raises GenericError: if cannot get a valid target_release """ - draft_release = buildinfo.get('release') - if not draft_release: - raise GenericError("'release' not found in %s" % buildinfo) - build_id = buildinfo.get('id') - if build_id: - build_id = str(build_id) - else: - raise GenericError("'id' not found in %s" % buildinfo) - match = re.match(DRAFT_RELEASE_PARSING_REGEX, draft_release) - if match: - if build_id != match.group('id'): - raise GenericError( - "buildinfo.id: %s doesn't match build id part: %s in buildinfo.release: %s" % - (build_id, match.group('id'), draft_release) - ) - return match.group('release') - else: + parts = draft_release.split(DRAFT_RELEASE_DELIMITER, 1) + if len(parts) != 2 or not parts[-1].startswith('draft_'): raise GenericError("draft release: %s is not in valid format" % draft_release) + return parts[0] def add_file_logger(logger, fn): diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index b66e820..8687e70 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6118,7 +6118,9 @@ def new_build(data, strict=False): # handle draft suffix in release if data.get('draft'): target_release = data['release'] - data['release'] = insert_data['release'] = koji.gen_draft_release(data) + data['release'] = insert_data['release'] = koji.gen_draft_release( + data['release'], data['id'] + ) # it's still possible to already have a build with the same nvr draft_nvr = dslice(data, ['name', 'version', 'release']) if find_build_id(draft_nvr): @@ -6428,7 +6430,7 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, d nvrinfo = buildinfo.copy() if draft: # for draft build, change release to target_release - nvrinfo['release'] = koji.parse_target_release(buildinfo) + nvrinfo['release'] = koji.parse_target_release(buildinfo['release']) srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % nvrinfo # either the sourcerpm field should match the build, or the filename # itself (for the srpm) @@ -7660,6 +7662,7 @@ def new_image_build(build_info): def new_typed_build(build_info, btype): """Mark build as a given btype""" # add here in case disabling draft build for non-rpm is missing before calling this + # TODO: remove it once draft build is expended to all types if btype != 'rpm': reject_draft(build_info) btype_id = lookup_name('btype', btype, strict=True)['id'] @@ -16024,7 +16027,7 @@ def _promote_build(build, user=None, strict=True, force=False): # get target release target_release = None try: - target_release = koji.parse_target_release(binfo) + target_release = koji.parse_target_release(old_release) except Exception: if strict: raise diff --git a/tests/test_cli/test_list_tagged.py b/tests/test_cli/test_list_tagged.py index a009fbd..7eea290 100644 --- a/tests/test_cli/test_list_tagged.py +++ b/tests/test_cli/test_list_tagged.py @@ -63,7 +63,7 @@ class TestCliListTagged(utils.CliTestCase): {'id': 2, 'name': 'packagename', 'version': 'version', - 'release': '2.el6#draft_2', + 'release': '2.el6,draft_2', 'nvr': 'n-v-r', 'draft': True, 'tag_name': 'tag', @@ -129,7 +129,7 @@ n-v-r tag owner def test_list_tagged_rpms(self, event_from_opts_mock, stdout): expected = """sigkey rpmA-0.0.1-1.el6.noarch sigkey rpmA-0.0.1-1.el6.x86_64 -sigkey rpmA-0.0.1-2.el6.x86_64 (#draft_2) +sigkey rpmA-0.0.1-2.el6.x86_64 (,draft_2) """ args = [self.tag, self.pkg, '--latest-n=3', '--rpms', '--sigs', '--arch=x86_64', '--arch=noarch'] @@ -150,7 +150,7 @@ sigkey rpmA-0.0.1-2.el6.x86_64 (#draft_2) def test_list_tagged_rpms_paths(self, event_from_opts_mock, stdout, os_path_exists, isdir): expected = """/mnt/koji/packages/packagename/version/1.el6/noarch/rpmA-0.0.1-1.el6.noarch.rpm /mnt/koji/packages/packagename/version/1.el6/x86_64/rpmA-0.0.1-1.el6.x86_64.rpm -/mnt/koji/packages/packagename/version/2.el6#draft_2/x86_64/rpmA-0.0.1-2.el6.x86_64.rpm +/mnt/koji/packages/packagename/version/2.el6,draft_2/x86_64/rpmA-0.0.1-2.el6.x86_64.rpm """ args = [self.tag, self.pkg, '--latest-n=3', '--rpms', '--arch=x86_64', '--paths'] diff --git a/tests/test_hub/test_import_rpm.py b/tests/test_hub/test_import_rpm.py index bd8acb0..314dc62 100644 --- a/tests/test_hub/test_import_rpm.py +++ b/tests/test_hub/test_import_rpm.py @@ -257,7 +257,7 @@ class TestImportRPM(unittest.TestCase): 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'name', 'version': 'version', - 'release': 'release#draft_12345', + 'release': 'release,draft_12345', 'id': 12345, 'draft': True, 'extra': { @@ -303,7 +303,7 @@ class TestImportRPM(unittest.TestCase): 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'name', 'version': 'version', - 'release': 'release#draft_12345', + 'release': 'release,draft_12345', 'id': 12345, 'draft': True } diff --git a/tests/test_hub/test_new_build.py b/tests/test_hub/test_new_build.py index 758da47..9e7bd74 100644 --- a/tests/test_hub/test_new_build.py +++ b/tests/test_hub/test_new_build.py @@ -176,7 +176,7 @@ class TestNewBuild(unittest.TestCase): 'id': 108, 'owner': 123, 'pkg_id': 54, - 'release': 'test_release#draft_108', + 'release': 'test_release,draft_108', 'source': None, 'start_time': 'NOW', 'state': 1, @@ -202,5 +202,5 @@ class TestNewBuild(unittest.TestCase): { 'name': 'test_name', 'version': 'test_version', - 'release': 'test_release#draft_108' + 'release': 'test_release,draft_108' }) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index 1653269..b323639 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -47,7 +47,7 @@ class TestPromoteBuild(unittest.TestCase): 'id': 1, 'name': 'foo', 'version': 'bar', - 'release': 'tgtrel#draft_1', + 'release': 'tgtrel,draft_1', 'nvr': 'testnvr', 'extra': { 'draft': { @@ -85,7 +85,7 @@ class TestPromoteBuild(unittest.TestCase): 'draft': { 'promoted': True, 'target_release': 'tgtrel', - 'old_release': 'tgtrel#draft_1', + 'old_release': 'tgtrel,draft_1', 'promotion_time': 'NOW', 'promotion_ts': self._now.timestamp(), 'promoter': self.user['name'] @@ -120,12 +120,13 @@ class TestPromoteBuild(unittest.TestCase): self.assertIsNone(ret) self.assertEqual(len(self.updates), 0) - def test_promote_build__target_release(self): + def test_promote_build_target_release(self): draft = { 'id': 1, 'name': 'foo', 'version': 'bar', - 'release': 'tgtrel#draft_2', + # bad delimiter + 'release': 'tgtrel@draft_1', 'extra': { 'draft': { 'promoted': False, @@ -145,7 +146,7 @@ class TestPromoteBuild(unittest.TestCase): self.exports.promoteBuild('a-regular-build', strict=True) self.assertEqual( str(cm.exception), - "buildinfo.id: 1 doesn't match build id part: 2 in buildinfo.release: tgtrel#draft_2" + "draft release: tgtrel@draft_1 is not in valid format" ) self.assertEqual(len(self.updates), 0) From 93c52c3eae1d47263a0b6b44900ca0ddded9843f Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:48 +0000 Subject: [PATCH 40/70] revert denying rpminfo.extra in import_rpm for draft --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 8687e70..d399f46 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6463,11 +6463,6 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, d if fileinfo is not None: extra = fileinfo.get('extra') if extra is not None: - if draft: - # simply deny draft - raise koji.GenericError( - f'When importing draft rpm, cannot specify extra in fileinfo: {fileinfo}' - ) rpminfo['extra'] = json.dumps(extra) koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo, From 444f4394c1a41af84a773d45abc76358868802e6 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:49 +0000 Subject: [PATCH 41/70] remove unnecessary reject_draft refs in HostExports As we should be able to guarantee that builder won't do that. --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index d399f46..6c79ff2 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -14919,7 +14919,6 @@ class HostExports(object): # ensure release is None so get_next_build will handle incrementing build_info['release'] = None data = build_info.copy() - reject_draft(data) data['task_id'] = task_id data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] @@ -15085,7 +15084,6 @@ class HostExports(object): task = Task(task_id) task.assertHost(host.id) data = build_info.copy() - reject_draft(data) data['task_id'] = task_id data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] @@ -15108,7 +15106,6 @@ class HostExports(object): task.assertHost(host.id) # build_info must contain name, version, and release data = build_info.copy() - reject_draft(data) data['task_id'] = task_id data['owner'] = task.getOwner() data['state'] = koji.BUILD_STATES['BUILDING'] diff --git a/vm/kojikamid.py b/vm/kojikamid.py index e35a39e..fe271e2 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -378,6 +378,7 @@ class WindowsBuild(object): buildinfo = self.server.getLatestBuild(self.build_tag, buildreq, self.task_opts.get('repo_id')) # don't allow draft build in buildrequires + # TODO: remove it when ready for win build if buildinfo.get('draft'): raise BuildError( # noqa: F821 "Draft build: %s is not supported in buildrequires" % buildinfo['nvr'] From a6a47c1d390ac9399578d092c766b70f0a46bc60 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:49 +0000 Subject: [PATCH 42/70] new_typed_build: do not reject draft --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 6c79ff2..4fa1a19 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -7656,10 +7656,6 @@ def new_image_build(build_info): def new_typed_build(build_info, btype): """Mark build as a given btype""" - # add here in case disabling draft build for non-rpm is missing before calling this - # TODO: remove it once draft build is expended to all types - if btype != 'rpm': - reject_draft(build_info) btype_id = lookup_name('btype', btype, strict=True)['id'] query = QueryProcessor(tables=['build_types'], columns=['build_id'], clauses=['build_id = %(build_id)i', 'btype_id = %(btype_id)i'], From 62e010d44eec9e215e5c35d2b1e9c650f1a92ee5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:49 +0000 Subject: [PATCH 43/70] reduce code duplication in mapInternalRPMs --- diff --git a/builder/kojid b/builder/kojid index 1eef7bd..a30a702 100755 --- a/builder/kojid +++ b/builder/kojid @@ -925,6 +925,7 @@ class BuildRoot(object): opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) rpmlist_path = os.path.join(self.get_repo_dir(), self.br_arch, 'rpmlist.jsonl') repo_rpms = {} + compat_mode = False try: with koji.openRemoteFile(rpmlist_path, **opts) as fo: for line in fo: @@ -935,9 +936,10 @@ class BuildRoot(object): if e.response.status_code == 404: self.logger.warning("Missing repo content file: %s", rpmlist_path) # TODO: remove this workaround once we can assume that repos contain this file - self.compat_map_internal_rpms(rpmlist) - return - raise + repo_rpms = self.repo_draft_rpms() + compat_mode = True + else: + raise fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" for rpm_info in rpmlist: if 'external_repo' in rpm_info: @@ -945,7 +947,9 @@ class BuildRoot(object): nvra = fmt % rpm_info data = repo_rpms.get(nvra) if not data: - self.logger.warning("%s not found in rpmlist.jsonl", nvra) + # happens a lot in compat mode because we only query for drafts + if not compat_mode: + self.logger.warning("%s not found in rpmlist.jsonl", nvra) continue # check payloadhash in case they are different elif data['payloadhash'] != rpm_info['payloadhash']: @@ -957,30 +961,13 @@ class BuildRoot(object): # set rpm id rpm_info['id'] = data['id'] - def compat_map_internal_rpms(self, rpmlist): + def repo_draft_rpms(self): drafts, draftbuilds = self.session.listTaggedRPMS( tag=self.repo_info['tag_id'], event=self.repo_info['create_event'], latest=True, draft=True) - fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" - content = {} - for draftinfo in drafts: - nvra = fmt % draftinfo - content[nvra] = draftinfo - for rpminfo in rpmlist: - nvra = fmt % rpminfo - data = content.get(nvra) - if data: - # draft rpm! - if data['payloadhash'] != rpminfo['payloadhash']: - raise koji.BuildrootError( - "RPM: %s: payloadhash: %s mismatch expected %s" - % (nvra, rpminfo['payloadhash'], data['payloadhash']) - ) - else: - # set rpm id - rpminfo['id'] = data['id'] + return drafts def path_without_to_within(self, path): """ From af36318ead0f7a015e748fc44cc10c7e8fb16b3b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:50 +0000 Subject: [PATCH 44/70] refactor get_rpm --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 4050fcc..58026dc 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -3551,28 +3551,18 @@ def anon_handle_list_builds(goptions, session, args): def anon_handle_rpminfo(goptions, session, args): "[info] Print basic information about an RPM" - usage = "usage: %prog rpminfo [options] [ ...]" + usage = "usage: %prog rpminfo [options] [ ...]" parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--buildroots", action="store_true", help="show buildroots the rpm was used in") - parser.add_option("--build", metavar="NVR|ID", - help="show the rpm(s) in the build") (options, args) = parser.parse_args(args) if len(args) < 1: parser.error("Please specify an RPM") - opts = {} - build = options.build - if options.build: - try: - build = int(build) - except ValueError: - pass - opts['build'] = build ensure_connection(session, goptions) error_hit = False for rpm in args: - info = session.getRPM(rpm, **opts) + info = session.getRPM(rpm) if info is None: warn("No such rpm: %s\n" % rpm) error_hit = True diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 4fa1a19..5c16ced 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -4609,11 +4609,12 @@ def _fix_rpm_row(row): _fix_archive_row = _fix_rpm_row -def get_rpm(rpminfo, strict=False, multi=False, build=None): +def get_rpm(rpminfo, strict=False, multi=False): """Get information about the specified RPM - rpminfo ma4666y be any one of the following: - - a int ID + rpminfo may be any one of the following: + - the rpm id as an int + - the rpm id as a string - a string N-V-R.A - a string N-V-R.A@location - a map containing 'name', 'version', 'release', and 'arch' @@ -4621,10 +4622,6 @@ def get_rpm(rpminfo, strict=False, multi=False, build=None): If specified, location should match the name of an external repo - If build is specfied, the rpm is limited to the build's - - build and location(not INTERNAL) is conflict because a rpm in - A map will be returned, with the following keys: - id - name @@ -4646,10 +4643,70 @@ def get_rpm(rpminfo, strict=False, multi=False, build=None): If there is no RPM with the given ID, None is returned, unless strict is True in which case an exception is raised - If more than one RPM matches, and multi is True, then a list of results is - returned. If multi is False, a single match is returned (an internal one if - possible). + This function is normally expected to return a single rpm. However, there + are cases where the given rpminfo could refer to multiple rpms. This is + because of nvra overlap involving: + * draft rpms + * external rpms + + If more than one RPM matches, then in the default case (multi=False), this function + will choose the best option in order of preference: + 1. internal non-draft rpms (nvras are unique within this subset) + 2. internal draft rpms (highest rpm id) + 3. external rpms (highest rpm id) + OTOH if multi is True, then all matching results are returned as a list """ + # we can look up by id or NVRA + data = None + if isinstance(rpminfo, int): + data = {'id': rpminfo} + elif isinstance(rpminfo, str): + # either nvra or id as a string + try: + data = {'id': int(rpminfo)} + except ValueError: + data = koji.parse_NVRA(rpminfo) + elif isinstance(rpminfo, dict): + data = rpminfo.copy() + else: + raise koji.GenericError("Invalid type for rpminfo: %r" % type(rpminfo)) + + rpms = _get_rpms(data) + if multi: + return rpms + + # otherwise make sure we have a single rpm + if not rpms: + if strict: + raise koji.GenericError("No such rpm: %r" % data) + return None + elif len(rpms) == 1: + return rpms[0] + else: + # pick our preferred, as described above + nondraft = None + draft = None + external = None + for rinfo in rpms: + if rinfo['external_repo_id']: + if external is None or rinfo['id'] > external['id']: + external = rinfo + elif rinfo['draft']: + if draft is None or rinfo['id'] > draft['id']: + draft = rinfo + else: + # rinfo is internal and nondraft + if nondraft: + # should not happen + # none of our selection options should result in more than one nondraft build + raise koji.GenericError("Multiple nondraft rpm matches for: %r" % data) + else: + nondraft = rinfo + return nondraft or draft or external + + +def _get_rpms(data): + """Helper function for get_rpm""" fields = ( ('rpminfo.id', 'id'), ('build_id', 'build_id'), @@ -4668,57 +4725,21 @@ def get_rpm(rpminfo, strict=False, multi=False, build=None): ('metadata_only', 'metadata_only'), ('extra', 'extra'), ) - # we can look up by id or NVRA - data = None - if isinstance(rpminfo, int): - data = {'id': rpminfo} - elif isinstance(rpminfo, str): - data = koji.parse_NVRA(rpminfo) - elif isinstance(rpminfo, dict): - data = rpminfo.copy() - else: - raise koji.GenericError("Invalid type for rpminfo: %r" % type(rpminfo)) clauses = [] if 'id' in data: clauses.append("rpminfo.id=%(id)s") else: clauses.append("rpminfo.name=%(name)s AND version=%(version)s " "AND release=%(release)s AND arch=%(arch)s") - # build and non-INTERNAL location (and multi=True) conflict in theory, - # but we just do the query and return None. - if build: - # strict=True as we treate build not found as an input error - data['build_id'] = find_build_id(build, strict=True) - clauses.append("rpminfo.build_id = %(build_id)s") - retry = False if 'location' in data: data['external_repo_id'] = get_external_repo_id(data['location'], strict=True) - clauses.append("""external_repo_id = %(external_repo_id)i""") - elif not multi: - # try to match internal first, otherwise first matching external - retry = True # if no internal match - orig_clauses = list(clauses) # copy - clauses.append("""external_repo_id = 0""") - + clauses.append("""external_repo_id = %(external_repo_id)s""") joins = ['external_repo ON rpminfo.external_repo_id = external_repo.id'] query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], tables=['rpminfo'], joins=joins, clauses=clauses, values=data, transform=_fix_rpm_row) - if multi: - return query.execute() - ret = query.executeOne() - if ret: - return ret - if retry: - # at this point we have just an NVRA with no internal match. Open it up to externals - query.clauses = orig_clauses - ret = query.executeOne() - if not ret: - if strict: - raise koji.GenericError("No such rpm: %r" % data) - return None - return ret + return query.execute() def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, diff --git a/tests/test_cli/test_rpminfo.py b/tests/test_cli/test_rpminfo.py index 8e7aebb..e4784f4 100644 --- a/tests/test_cli/test_rpminfo.py +++ b/tests/test_cli/test_rpminfo.py @@ -64,7 +64,7 @@ class TestRpminfo(utils.CliTestCase): 'version': '1.1', 'payloadhash': 'b2b95550390e5f213fc25f33822425f7', 'size': 7030}] - self.error_format = """Usage: %s rpminfo [options] [ ...] + self.error_format = """Usage: %s rpminfo [options] [ ...] (Specify the --help global option for a list of other help options) %s: error: {message} @@ -171,42 +171,6 @@ Used in 1 buildroots: arches='src') - @mock.patch('sys.stdout', new_callable=StringIO) - def test_handle_rpminfo_with_build(self, stdout): - rpm_nvra = 'test-rpm-1.1-11.noarch' - self.session.getBuildroot.return_value = self.buildroot_info - self.session.listBuildroots.return_value = [self.buildroot_info] - self.session.getBuild.return_value = self.buildinfo - self.session.getRPM.return_value = self.getrpminfo - self.session.listRPMs.return_value = self.listrpminfos - expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] -Build: test-rpm-1.1-11 [1] -RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm -SRPM: 7:test-rpm-1.1-11 [290] -SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm -Built: Tue, 16 Mar 2021 06:56:49 UTC -SIGMD5: b2b95550390e5f213fc25f33822425f7 -Size: 7030 -Build ID: 1 -Buildroot: 3 (tag test-tag, arch x86_64, repo 2) -Build Host: kojibuilder -Build Task: 10 -Used in 1 buildroots: - id build tag arch build host - -------- ---------------------------- -------- ----------------------------- - 3 test-tag x86_64 kojibuilder -""" - - anon_handle_rpminfo(self.options, self.session, ['--buildroot', '--build', 'any', rpm_nvra]) - self.assert_console_message(stdout, expected_output) - self.session.getBuildroot.assert_called_once_with(self.getrpminfo['buildroot_id']) - self.session.listBuildroots.assert_called_once_with(queryOpts={'order': 'buildroot.id'}, - rpmID=self.getrpminfo['id']) - self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) - self.session.getRPM.assert_called_once_with(rpm_nvra, build='any') - self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], - arches='src') - def test_rpminfo_without_option(self): arguments = [] self.assert_system_exit( @@ -223,11 +187,10 @@ Used in 1 buildroots: def test_rpminfo_help(self): self.assert_help( anon_handle_rpminfo, - """Usage: %s rpminfo [options] [ ...] + """Usage: %s rpminfo [options] [ ...] (Specify the --help global option for a list of other help options) Options: - -h, --help show this help message and exit - --buildroots show buildroots the rpm was used in - --build=NVR|ID show the rpm(s) in the build + -h, --help show this help message and exit + --buildroots show buildroots the rpm was used in """ % self.progname) diff --git a/tests/test_hub/test_getRPM.py b/tests/test_hub/test_getRPM.py index 1e19570..ef15d46 100644 --- a/tests/test_hub/test_getRPM.py +++ b/tests/test_hub/test_getRPM.py @@ -29,7 +29,7 @@ class TestGetRPM(DBQueryTestCase): def test_rpm_info_int(self): rpminfo = 123 - self.qp_execute_one_return_value = {'rpminfo.id': 123} + self.qp_execute_return_value = [{'rpminfo.id': 123}] result = kojihub.get_rpm(rpminfo) self.assertEqual(result, {'rpminfo.id': 123}) @@ -40,7 +40,7 @@ class TestGetRPM(DBQueryTestCase): 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) - self.assertEqual(query.clauses, ['external_repo_id = 0', "rpminfo.id=%(id)s"]) + self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"]) self.assertEqual(query.joins, ['external_repo ON rpminfo.external_repo_id = external_repo.id']) self.assertEqual(query.values, {'id': rpminfo}) @@ -116,27 +116,7 @@ class TestGetRPM(DBQueryTestCase): 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, - ["external_repo_id = %(external_repo_id)i", "rpminfo.id=%(id)s"]) - self.assertEqual(query.joins, - ['external_repo ON rpminfo.external_repo_id = external_repo.id']) - self.assertEqual(query.values, rpminfo_data) - - def test_rpm_info_with_build(self): - rpminfo = {'id': 123, 'name': 'testrpm-1.23-4.x86_64.rpm', 'build_id': 101} - self.find_build_id.return_value = 101 - rpminfo_data = rpminfo.copy() - - kojihub.get_rpm(rpminfo, multi=True, build='any') - - self.assertEqual(len(self.queries), 1) - query = self.queries[0] - self.assertEqual(query.tables, ['rpminfo']) - columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', - 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] - self.assertEqual(set(query.columns), set(columns)) - self.assertEqual(query.clauses, - ["rpminfo.build_id = %(build_id)s", "rpminfo.id=%(id)s"]) + ["external_repo_id = %(external_repo_id)s", "rpminfo.id=%(id)s"]) self.assertEqual(query.joins, ['external_repo ON rpminfo.external_repo_id = external_repo.id']) self.assertEqual(query.values, rpminfo_data) From a945523a1b1c795c5ce36fd3484a4f344762fa20 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:51 +0000 Subject: [PATCH 45/70] fix get_rpm call --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 5c16ced..7049db9 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -14302,7 +14302,7 @@ class BuildRoot(object): data = add_external_rpm(an_rpm, location, strict=False) # will add if missing, compare if not else: - data = get_rpm(an_rpm, strict=True, build=an_rpm.get('build', None)) + data = get_rpm(an_rpm, strict=True) rpm_id = data['id'] if update and rpm_id in current: # ignore duplicate packages for updates From deaa6a01e816fd0cb01fc5061a6efc182b90b489 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:51 +0000 Subject: [PATCH 46/70] gen_draft_release: clearer argument naming not to mask builtin id function --- diff --git a/koji/__init__.py b/koji/__init__.py index 113dd85..334b403 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -298,7 +298,7 @@ DEFAULT_AUTH_TIMEOUT = 60 # draft release constants DRAFT_RELEASE_DELIMITER = ',' -DRAFT_RELEASE_FORMAT = '{release}' + DRAFT_RELEASE_DELIMITER + 'draft_{id}' +DRAFT_RELEASE_FORMAT = '{target_release}' + DRAFT_RELEASE_DELIMITER + 'draft_{build_id}' # BEGIN kojikamid dup # @@ -3930,15 +3930,15 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): return walker.walk() -def gen_draft_release(release, id): +def gen_draft_release(target_release, build_id): """Generate draft_release based on input build information Currently, it's generated as {target_release},draft_{build_id} - :param str release: target "release", which is the release part of rpms' nvra, + :param str target_release: target "release", which is the release part of rpms' nvra, and will be the release of the build the draft build is going to be promoted to. - :param int id: the build "id" part in draft_release (it's unchanged so it can be used to + :param int build_id: the build "id" part in draft_release (it's unchanged so it can be used to keep the uniqueness of build NVR) :return: draft release :rtype: str From e32c71c100d5180bb941098123bdc119b19b92a2 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:52 +0000 Subject: [PATCH 47/70] remove draft opts in import_build and completeBuild --- diff --git a/builder/kojid b/builder/kojid index a30a702..eb31a51 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1204,8 +1204,7 @@ class BuildTask(BaseTaskHandler): # scratch builds do not get imported self.session.host.moveBuildToScratch(self.id, srpm, rpms, logs=logs) else: - self.session.host.completeBuild(self.id, build_id, srpm, rpms, brmap, logs=logs, - draft=opts.get('draft') or False) + self.session.host.completeBuild(self.id, build_id, srpm, rpms, brmap, logs=logs) except (SystemExit, ServerExit, KeyboardInterrupt): # we do not trap these raise @@ -2356,8 +2355,7 @@ class WrapperRPMTask(BaseBuildTask): try: self.session.host.completeBuild(self.id, self.new_build_id, relsrpm, relrpms, brmap, - {'noarch': rellogs}, - draft=opts.get('draft') or False) + {'noarch': rellogs}) except (SystemExit, ServerExit, KeyboardInterrupt): raise except Exception: diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 7049db9..c6063f9 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6273,7 +6273,7 @@ def check_noarch_rpms(basepath, rpms, logs=None): return result -def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, draft=False): +def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None): """Import a build into the database (single transaction) Files must be uploaded and specified with path relative to the workdir @@ -6283,9 +6283,6 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, brmap - dictionary mapping [s]rpms to buildroot ids task_id - associate the build with a task build_id - build is a finalization of existing entry - draft - If True and - - build_id: None, to create a draft build - Not None, the build_id must be a draft build with a valid release """ if brmap is None: brmap = {} @@ -6293,7 +6290,6 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, convert_value(brmap, cast=dict, check_only=True) convert_value(srpm, cast=str, check_only=True) convert_value(rpms, cast=list, check_only=True) - draft = bool(draft) koji.plugin.run_callbacks('preImport', type='build', srpm=srpm, rpms=rpms, brmap=brmap, task_id=task_id, build_id=build_id, build=None, logs=logs) uploadpath = koji.pathinfo.work() @@ -6337,7 +6333,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, build['volume_name'] = vol['name'] if build_id is None: - build['draft'] = draft + # it is always a non-draft build build_id = new_build(build) binfo = get_build(build_id, strict=True) new_typed_build(binfo, 'rpm') @@ -6350,13 +6346,8 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=binfo) - if binfo.get('draft') != draft: - raise koji.GenericError( - f"draft propetry of build: {binfo['nvr']} is not correct. Expected: {draft}," - f" got {binfo['draft']}" - ) - if draft: - build['release'] = koji.DRAFT_RELEASE_FORMAT.format(**build) + if binfo.get('draft'): + build['release'] = koji.gen_draft_release(build['release'], build['id']) for key in ('name', 'version', 'release', 'epoch', 'task_id'): if build[key] != binfo[key]: raise koji.GenericError( @@ -6378,7 +6369,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, # now to handle the individual rpms for relpath in [srpm] + rpms: fn = "%s/%s" % (uploadpath, relpath) - rpminfo = import_rpm(fn, binfo, brmap.get(relpath), draft=draft) + rpminfo = import_rpm(fn, binfo, brmap.get(relpath), draft=binfo['draft']) import_rpm_file(fn, binfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) if logs: @@ -14866,14 +14857,14 @@ class HostExports(object): new_typed_build(binfo, 'rpm') return build_id - def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None, draft=False): + def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None): """Import final build contents into the database""" # sanity checks host = Host() host.verify() task = Task(task_id) task.assertHost(host.id) - result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs, draft=draft) + result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs) build_notification(task_id, build_id) return result diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index a81d18f..ddef0e9 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -82,6 +82,7 @@ class TestImportBuild(unittest.TestCase): 'version': 'version', 'release': 'release', 'id': 12345, + 'draft': False } # get_build called once to check for existing, # if it doesn't exist, called another time after creating From 23f8a4f0093b0b14aee2abe1e42aff485f6a6a3f Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:52 +0000 Subject: [PATCH 48/70] promoteBuild: readable error msg --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index c6063f9..d316a06 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -16012,7 +16012,7 @@ def _promote_build(build, user=None, strict=True, force=False): return None if not binfo.get('draft'): if strict: - raise koji.GenericError(f'Not a draft build: {binfo}') + raise koji.GenericError(f"Not a draft build: {binfo['nvr']}") else: return None state = koji.BUILD_STATES[binfo['state']] @@ -16043,7 +16043,9 @@ def _promote_build(build, user=None, strict=True, force=False): old_build = get_build(target_build) if old_build: if strict: - raise koji.GenericError(f'Target build already exists: {old_build}') + raise koji.GenericError( + f"Cannot promote to an existing target build: {old_build['nvr']}(#{old_build['id']})" + ) else: return None From 1fa665f24032ac45e7e021f6291eddceee15bfaa Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:53 +0000 Subject: [PATCH 49/70] reject_draft: remove strict option --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index d316a06..17f31af 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -7047,7 +7047,7 @@ class CG_Importer(object): self.log_warning("IGNORING unmatched rpm component: %r" % comp) return None # TODO: we should consider how to handle them once draft build is enabled for CG - reject_draft(rinfo, is_rpm=True, strict=True) + reject_draft(rinfo, is_rpm=True) if rinfo['payloadhash'] != comp['sigmd5']: # XXX - this is a temporary workaround until we can better track external refs self.log_warning("IGNORING rpm component (md5 mismatch): %r" % comp) @@ -15940,7 +15940,7 @@ def create_rpm_checksum(rpm_id, sigkey, chsum_dict): insert.execute() -def reject_draft(data, is_rpm=False, error=None, strict=True): +def reject_draft(data, is_rpm=False, error=None): """block draft build/rpm TODO: remove this once draft build is open for all build types @@ -15949,11 +15949,8 @@ def reject_draft(data, is_rpm=False, error=None, strict=True): :param bool is_rpm: indicates data is rpm or build (true/false) in default error msg :param koji.GenericError error: the error raised if not a draft build, defaults to None to raise the default "unsupported" error - :param bool strict: if True will raise an error if it's draft, else return the draft value. :raises error: default or specified by input error when draft==True in data """ - if not strict: - return bool(data.get('draft')) if data.get('draft'): if error is None: entry_type = 'rpm' if is_rpm else 'build' From 3dbe136187d0cb0a175911eeb803731dbae8b2d5 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:53 +0000 Subject: [PATCH 50/70] promoteBuild: remove strict option --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 17f31af..49b2faa 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -13680,7 +13680,7 @@ class RootExports(object): koji.plugin.run_callbacks('postBuildStateChange', attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) - def promoteBuild(self, build, strict=True, force=False): + def promoteBuild(self, build, force=False): """Promote a draft build to a regular build. - The build type is limited to rpm so far. @@ -13692,20 +13692,17 @@ class RootExports(object): :param build: A build ID (int), a NVR (string), or a dict containing "name", "version" and "release" of a draft build :type build: int, str, dict - :param bool strict: Whether raising Error (default) - or depress it by return None :param bool force: If False (default), Koji will check this operation against the draft_promotion hub policy. If hub policy does not allow the current user to promote the draft build, - then this method will raise an error (strict=True) or return None - (strict=False). + then this method will raise an error If True, then this method will bypass hub policy settings. Only admin users can set force to True. - :returns: latest build info, or None if any failure with strict=False + :returns: latest build info :rtype: dict """ context.session.assertLogin() - return _promote_build(build, strict=strict, force=force) + return _promote_build(build, force=force) def count(self, methodName, *args, **kw): """Execute the XML-RPC method with the given name and count the results. @@ -16002,53 +15999,29 @@ def _clean_draft_link(promoted_build): os.unlink(draft_builddir) -def _promote_build(build, user=None, strict=True, force=False): +def _promote_build(build, user=None, force=False): """promote a draft build to a regular one""" - binfo = get_build(build, strict=strict) - if not binfo: - return None + binfo = get_build(build, strict=True) if not binfo.get('draft'): - if strict: - raise koji.GenericError(f"Not a draft build: {binfo['nvr']}") - else: - return None + raise koji.GenericError(f"Not a draft build: {binfo['nvr']}") state = koji.BUILD_STATES[binfo['state']] if state != 'COMPLETE': - if strict: - raise koji.GenericError(f"Cannot promote build - {binfo['nvr']}." - f' Reason: state ({state}) is not COMPLETE.') - else: - return None - old_release = binfo['release'] + raise koji.GenericError(f"Cannot promote build - {binfo['nvr']}." + f' Reason: state ({state}) is not COMPLETE.') - # get target release - target_release = None - try: - target_release = koji.parse_target_release(old_release) - except Exception: - if strict: - raise - if not target_release: - if strict: - # should never happen - raise koji.GenericError('Cannot generate target_release') - return None + old_release = binfo['release'] + target_release = koji.parse_target_release(old_release) # drop id to get build by NVR target_build = dslice(binfo, ['name', 'version']) target_build['release'] = target_release old_build = get_build(target_build) if old_build: - if strict: - raise koji.GenericError( - f"Cannot promote to an existing target build: {old_build['nvr']}(#{old_build['id']})" - ) - else: - return None + raise koji.GenericError( + f"Cannot promote to an existing target build: {old_build['nvr']}(#{old_build['id']})" + ) - user = get_user(user, strict=strict) - if not user and not strict: - return None + user = get_user(user, strict=True) # policy checks policy_data = { @@ -16082,7 +16055,7 @@ def _promote_build(build, user=None, strict=True, force=False): update.set(draft=False, release=target_release, extra=extra) update.execute() - new_binfo = get_build(binfo['id'], strict=strict) + new_binfo = get_build(binfo['id'], strict=True) move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) ensure_volume_symlink(new_binfo) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index b323639..624b697 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -93,7 +93,7 @@ class TestPromoteBuild(unittest.TestCase): } ) - ret = self.exports.promoteBuild('a-draft-build', strict=True) + ret = self.exports.promoteBuild('a-draft-build') self.assertEqual(ret, self.new_build) self.assertEqual(len(self.updates), 1) update = self.updates[0] @@ -109,15 +109,11 @@ class TestPromoteBuild(unittest.TestCase): ) def test_promote_build_not_draft(self): - self.get_build.return_value = {'draft': False} + self.get_build.return_value = {'draft': False, 'nvr': 'testnvr'} with self.assertRaises(koji.GenericError) as cm: - self.exports.promoteBuild('a-regular-build', strict=True) - self.assertEqual(str(cm.exception), "Not a draft build: {'draft': False}") - self.assertEqual(len(self.updates), 0) - - ret = self.exports.promoteBuild('a-regular-build', strict=False) - self.assertIsNone(ret) + self.exports.promoteBuild('a-regular-build') + self.assertEqual(str(cm.exception), "Not a draft build: testnvr") self.assertEqual(len(self.updates), 0) def test_promote_build_target_release(self): @@ -143,17 +139,13 @@ class TestPromoteBuild(unittest.TestCase): self.get_build.return_value = draft with self.assertRaises(koji.GenericError) as cm: - self.exports.promoteBuild('a-regular-build', strict=True) + self.exports.promoteBuild('a-regular-build') self.assertEqual( str(cm.exception), "draft release: tgtrel@draft_1 is not in valid format" ) self.assertEqual(len(self.updates), 0) - ret = self.exports.promoteBuild('a-regular-build', strict=False) - self.assertIsNone(ret) - self.assertEqual(len(self.updates), 0) - def test_promote_build_not_completed(self): draft = { 'id': 1, @@ -176,35 +168,29 @@ class TestPromoteBuild(unittest.TestCase): self.get_build.return_value = draft with self.assertRaises(koji.GenericError) as cm: - self.exports.promoteBuild('a-regular-build', strict=True) + self.exports.promoteBuild('a-regular-build') self.assertEqual( str(cm.exception), f"Cannot promote build - {draft['nvr']}. Reason: state (BUILDING) is not COMPLETE." ) self.assertEqual(len(self.updates), 0) - ret = self.exports.promoteBuild('a-regular-build', strict=False) - self.assertIsNone(ret) - self.assertEqual(len(self.updates), 0) - def test_promote_build_target_build_exists(self): old = { - 'id': 'any' + 'id': 'any', + 'nvr': 'oldnvr' } self.get_build.side_effect = [self.draft_build, old] with self.assertRaises(koji.GenericError) as cm: - self.exports.promoteBuild('a-regular-build', strict=True) - self.assertEqual(str(cm.exception), f"Target build already exists: {old}") + self.exports.promoteBuild('a-regular-build') + self.assertEqual( + str(cm.exception), + "Cannot promote to an existing target build: oldnvr(#any)" + ) self.assertEqual(len(self.updates), 0) self.get_build.assert_called_with({ 'name': 'foo', 'version': 'bar', 'release': 'tgtrel' }) - - self.get_build.reset_mock() - self.get_build.side_effect = [self.draft_build, old] - ret = self.exports.promoteBuild('a-regular-build', strict=False) - self.assertIsNone(ret) - self.assertEqual(len(self.updates), 0) From 8a64c93d054cb6d15d63bf222c9f5cae78b036e1 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:54 +0000 Subject: [PATCH 51/70] promoteBuild: reformat error messages --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 49b2faa..195e8bc 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -16002,12 +16002,12 @@ def _clean_draft_link(promoted_build): def _promote_build(build, user=None, force=False): """promote a draft build to a regular one""" binfo = get_build(build, strict=True) + err_fmt = f"Cannot promote build {binfo['nvr']}. Reason: {{}}" if not binfo.get('draft'): - raise koji.GenericError(f"Not a draft build: {binfo['nvr']}") + raise koji.GenericError(err_fmt.format("Not a draft build")) state = koji.BUILD_STATES[binfo['state']] if state != 'COMPLETE': - raise koji.GenericError(f"Cannot promote build - {binfo['nvr']}." - f' Reason: state ({state}) is not COMPLETE.') + raise koji.GenericError(err_fmt.format(f'state ({state}) is not COMPLETE.')) old_release = binfo['release'] target_release = koji.parse_target_release(old_release) @@ -16015,11 +16015,11 @@ def _promote_build(build, user=None, force=False): # drop id to get build by NVR target_build = dslice(binfo, ['name', 'version']) target_build['release'] = target_release - old_build = get_build(target_build) + old_build = get_build(target_build, strict=False) if old_build: - raise koji.GenericError( - f"Cannot promote to an existing target build: {old_build['nvr']}(#{old_build['id']})" - ) + raise koji.GenericError(err_fmt.format( + f"Target build exists: {old_build['nvr']}(#{old_build['id']})" + )) user = get_user(user, strict=True) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index 624b697..a3b05f3 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -113,7 +113,10 @@ class TestPromoteBuild(unittest.TestCase): with self.assertRaises(koji.GenericError) as cm: self.exports.promoteBuild('a-regular-build') - self.assertEqual(str(cm.exception), "Not a draft build: testnvr") + self.assertEqual( + str(cm.exception), + "Cannot promote build testnvr. Reason: Not a draft build" + ) self.assertEqual(len(self.updates), 0) def test_promote_build_target_release(self): @@ -123,6 +126,7 @@ class TestPromoteBuild(unittest.TestCase): 'version': 'bar', # bad delimiter 'release': 'tgtrel@draft_1', + 'nvr': 'testnvr', 'extra': { 'draft': { 'promoted': False, @@ -171,7 +175,7 @@ class TestPromoteBuild(unittest.TestCase): self.exports.promoteBuild('a-regular-build') self.assertEqual( str(cm.exception), - f"Cannot promote build - {draft['nvr']}. Reason: state (BUILDING) is not COMPLETE." + f"Cannot promote build {draft['nvr']}. Reason: state (BUILDING) is not COMPLETE." ) self.assertEqual(len(self.updates), 0) @@ -186,11 +190,11 @@ class TestPromoteBuild(unittest.TestCase): self.exports.promoteBuild('a-regular-build') self.assertEqual( str(cm.exception), - "Cannot promote to an existing target build: oldnvr(#any)" + "Cannot promote build testnvr. Reason: Target build exists: oldnvr(#any)" ) self.assertEqual(len(self.updates), 0) self.get_build.assert_called_with({ 'name': 'foo', 'version': 'bar', 'release': 'tgtrel' - }) + }, strict=False) From 4cc4233145c813583f5bf7355196b753bae6377d Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:55 +0000 Subject: [PATCH 52/70] make promoteBuild as staticmethod(_promote_build) --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 195e8bc..86f2c3d 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -10222,6 +10222,104 @@ def importImageInternal(task_id, build_info, imgdata): build=build_info, fullpath=fullpath) +def _promote_build(build, force=False): + """Promote a draft build to a regular build. + + - The build type is limited to rpm so far. + - The promoting action cannot be revoked. + - The release wil be changed to the target one, so build_id isn't changed + - buildpath will be changed as well and the old build path will symlink + to the new one, so both paths still will be existing until deleted. + + :param build: A build ID (int), a NVR (string), or a dict containing + "name", "version" and "release" of a draft build + :type build: int, str, dict + :param bool force: If False (default), Koji will check this + operation against the draft_promotion hub policy. If hub + policy does not allow the current user to promote the draft build, + then this method will raise an error. + If True, then this method will bypass hub policy settings. + Only admin users can set force to True. + :returns: latest build info + :rtype: dict + """ + context.session.assertLogin() + user = get_user(context.session.user_id) + + binfo = get_build(build, strict=True) + err_fmt = f"Cannot promote build {binfo['nvr']}. Reason: {{}}" + if not binfo.get('draft'): + raise koji.GenericError(err_fmt.format("Not a draft build")) + state = koji.BUILD_STATES[binfo['state']] + if state != 'COMPLETE': + raise koji.GenericError(err_fmt.format(f'state ({state}) is not COMPLETE.')) + + old_release = binfo['release'] + target_release = koji.parse_target_release(old_release) + + # drop id to get build by NVR + target_build = dslice(binfo, ['name', 'version']) + target_build['release'] = target_release + old_build = get_build(target_build, strict=False) + if old_build: + raise koji.GenericError(err_fmt.format( + f"Target build exists: {old_build['nvr']}(#{old_build['id']})" + )) + + # policy checks + policy_data = { + 'build': binfo['id'], + 'target_release': target_release, + 'user_id': user['id'] + } + assert_policy('draft_promotion', policy_data, force=force) + + koji.plugin.run_callbacks( + 'preBuildPromote', + draft_release=old_release, + target_release=target_release, + build=binfo, + user=user + ) + + # temp solution with python generated time, maybe better to be an event? + now = datetime.datetime.now() + + extra = binfo.get('extra') or {} + draft_info = extra.setdefault('draft', {}) + draft_info['promoted'] = True + draft_info['old_release'] = old_release + draft_info['promotion_time'] = encode_datetime(now) + draft_info['promotion_ts'] = now.timestamp() + draft_info['promoter'] = user['name'] + extra = json.dumps(extra) + + update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) + update.set(draft=False, release=target_release, extra=extra) + update.execute() + + new_binfo = get_build(binfo['id'], strict=True) + move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) + ensure_volume_symlink(new_binfo) + + # apply volume policy in case it's changed by release update. + apply_volume_policy(new_binfo, strict=False) + + # adding DRAFT_PROMOTION for kojira, + # as the latest promoted build should be that latest one. + for tag in list_tags(build=binfo['id']): + set_tag_update(tag['id'], 'DRAFT_PROMOTION') + + koji.plugin.run_callbacks( + 'postBuildPromote', + draft_release=old_release, + target_release=target_release, + build=new_binfo, + user=user + ) + return new_binfo + + def _delete_event_id(): """Helper function to bump event""" try: @@ -13680,29 +13778,7 @@ class RootExports(object): koji.plugin.run_callbacks('postBuildStateChange', attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) - def promoteBuild(self, build, force=False): - """Promote a draft build to a regular build. - - - The build type is limited to rpm so far. - - The promoting action cannot be revoked. - - The release wil be changed to the target one, so build_id isn't changed - - buildpath will be changed as well and the old build path will symlink - to the new one, so both paths still will be existing until deleted. - - :param build: A build ID (int), a NVR (string), or a dict containing - "name", "version" and "release" of a draft build - :type build: int, str, dict - :param bool force: If False (default), Koji will check this - operation against the draft_promotion hub policy. If hub - policy does not allow the current user to promote the draft build, - then this method will raise an error - If True, then this method will bypass hub policy settings. - Only admin users can set force to True. - :returns: latest build info - :rtype: dict - """ - context.session.assertLogin() - return _promote_build(build, force=force) + promoteBuild = staticmethod(_promote_build) def count(self, methodName, *args, **kw): """Execute the XML-RPC method with the given name and count the results. @@ -15997,81 +16073,3 @@ def _clean_draft_link(promoted_build): # only unlink whe its a symlink. if os.path.islink(draft_builddir): os.unlink(draft_builddir) - - -def _promote_build(build, user=None, force=False): - """promote a draft build to a regular one""" - binfo = get_build(build, strict=True) - err_fmt = f"Cannot promote build {binfo['nvr']}. Reason: {{}}" - if not binfo.get('draft'): - raise koji.GenericError(err_fmt.format("Not a draft build")) - state = koji.BUILD_STATES[binfo['state']] - if state != 'COMPLETE': - raise koji.GenericError(err_fmt.format(f'state ({state}) is not COMPLETE.')) - - old_release = binfo['release'] - target_release = koji.parse_target_release(old_release) - - # drop id to get build by NVR - target_build = dslice(binfo, ['name', 'version']) - target_build['release'] = target_release - old_build = get_build(target_build, strict=False) - if old_build: - raise koji.GenericError(err_fmt.format( - f"Target build exists: {old_build['nvr']}(#{old_build['id']})" - )) - - user = get_user(user, strict=True) - - # policy checks - policy_data = { - 'build': binfo['id'], - 'target_release': target_release, - 'user_id': user['id'] - } - assert_policy('draft_promotion', policy_data, force=force) - - koji.plugin.run_callbacks( - 'preBuildPromote', - draft_release=old_release, - target_release=target_release, - build=binfo, - user=user - ) - - # temp solution with python generated time, maybe better to be an event? - now = datetime.datetime.now() - - extra = binfo.get('extra') or {} - draft_info = extra.setdefault('draft', {}) - draft_info['promoted'] = True - draft_info['old_release'] = old_release - draft_info['promotion_time'] = encode_datetime(now) - draft_info['promotion_ts'] = now.timestamp() - draft_info['promoter'] = user['name'] - extra = json.dumps(extra) - - update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) - update.set(draft=False, release=target_release, extra=extra) - update.execute() - - new_binfo = get_build(binfo['id'], strict=True) - move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) - ensure_volume_symlink(new_binfo) - - # apply volume policy in case it's changed by release update. - apply_volume_policy(new_binfo, strict=False) - - # adding DRAFT_PROMOTION for kojira, - # as the latest promoted build should be that latest one. - for tag in list_tags(build=binfo['id']): - set_tag_update(tag['id'], 'DRAFT_PROMOTION') - - koji.plugin.run_callbacks( - 'postBuildPromote', - draft_release=old_release, - target_release=target_release, - build=new_binfo, - user=user - ) - return new_binfo From 8c32aa37cbbf4f9f788a175e8294b1560f8191ae Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:55 +0000 Subject: [PATCH 53/70] drop draft options from importRPM --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 86f2c3d..e0281ab 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6369,7 +6369,7 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) # now to handle the individual rpms for relpath in [srpm] + rpms: fn = "%s/%s" % (uploadpath, relpath) - rpminfo = import_rpm(fn, binfo, brmap.get(relpath), draft=binfo['draft']) + rpminfo = import_rpm(fn, binfo, brmap.get(relpath)) import_rpm_file(fn, binfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) if logs: @@ -6387,17 +6387,11 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) return binfo -def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, draft=False): +def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): """Import a single rpm into the database Designed to be called from import_build. """ - # draft option must be sane - if buildinfo and draft != buildinfo.get('draft', False): - raise koji.GenericError( - f'draft property: {buildinfo["draft"]} of build: {buildinfo["id"]} mismatch,' - f' {draft} is expected' - ) if not os.path.exists(fn): raise koji.GenericError("No such file: %s" % fn) @@ -6405,6 +6399,7 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, d hdr = koji.get_rpm_header(fn) rpminfo = koji.get_header_fields(hdr, ['name', 'version', 'release', 'epoch', 'sourcepackage', 'arch', 'buildtime', 'sourcerpm']) + draft = True if buildinfo and buildinfo.get('draft') else False rpminfo['draft'] = draft if rpminfo['sourcepackage'] == 1: rpminfo['arch'] = "src" @@ -6416,21 +6411,18 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, d raise koji.GenericError("bad filename: %s (expected %s)" % (basename, expected)) if buildinfo is None: - # figure it out for ourselves + # This only happens when we're called from importRPM + if draft: + # shouldn't happen with current code + raise koji.GenericError('rpm import is not supported for draft builds') if rpminfo['sourcepackage'] == 1: - if not draft: - # we only check if nvr already exists for non-draft - buildinfo = get_build(rpminfo, strict=False) + buildinfo = get_build(rpminfo, strict=False) if not buildinfo: # create a new build build_id = new_build(rpminfo) # we add the rpm build type below buildinfo = get_build(build_id, strict=True) else: - if draft: - raise koji.GenericError( - f'Cannot import draft rpm: {basename} without specifying a build' - ) # figure it out from sourcerpm string buildinfo = get_build(koji.parse_NVRA(rpminfo['sourcerpm'])) if buildinfo is None: @@ -11194,24 +11186,17 @@ class RootExports(object): reject_draft(build) new_image_build(build) - def importRPM(self, path, basename, build=None, draft=False): + def importRPM(self, path, basename): """Import an RPM into the database. The file must be uploaded first. """ context.session.assertPerm('admin') - if build: - # can get unique nvr from rpm header when importing to regular build - if not draft: - raise koji.ParameterError( - "Only support specifying build when importing a draft rpm" - ) - build = get_build(build, strict=True) uploadpath = koji.pathinfo.work() fn = "%s/%s/%s" % (uploadpath, path, basename) if not os.path.exists(fn): raise koji.GenericError("No such file: %s" % fn) - rpminfo = import_rpm(fn, buildinfo=build, draft=draft) + rpminfo = import_rpm(fn) import_rpm_file(fn, rpminfo['build'], rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) for tag in list_tags(build=rpminfo['build_id']): From cf387e5c73e7af65d7dc3f22774368f3486f3a57 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:56 +0000 Subject: [PATCH 54/70] import_build without a build_id should never happen --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index e0281ab..e6f6b31 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6333,7 +6333,8 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) build['volume_name'] = vol['name'] if build_id is None: - # it is always a non-draft build + # This should not be possible + logger.error('Calling import_build without a build_id is deprecated') build_id = new_build(build) binfo = get_build(build_id, strict=True) new_typed_build(binfo, 'rpm') From cbd9a10dfce6edfe0cd84bb437ece4f07ec71339 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:56 +0000 Subject: [PATCH 55/70] delete all build symlinks when deleting a build --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index e6f6b31..192a056 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -8815,7 +8815,7 @@ def _delete_build(binfo): update = UpdateProcessor('build', values=values, clauses=['id=%(build_id)i'], data={'state': st_deleted}) update.execute() - _clean_draft_link(binfo) + _delete_build_symlinks(binfo) # now clear the build dir builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): @@ -8825,6 +8825,37 @@ def _delete_build(binfo): attribute='state', old=st_old, new=st_deleted, info=binfo) +def _delete_build_symlinks(binfo): + """Remove symlinks pointing to the build + + These include the symlink created by ensure_volume_symlink and the + symlink created when draft builds are promoted. + """ + + base_vol = lookup_name('volume', 'DEFAULT', strict=True) + if binfo['volume_id'] != base_vol['id']: + # remove the link created by ensure_volume_symlinks + base_binfo = binfo.copy() + base_binfo['volume_id'] = base_vol['id'] + base_binfo['volume_name'] = base_vol['name'] + basedir = koji.pathinfo.build(base_binfo) + if os.path.islink(basedir): + os.unlink(basedir) + + if not binfo['draft']: + # if the build isn't a draft, it may once have been + draft_release = koji.gen_draft_release(binfo['release'], binfo['id']) + for check_vol in list_volumes(): + # the build could have been on any volume when promoted + check_binfo = binfo.copy() + check_binfo['volume_id'] = check_vol['id'] + check_binfo['volume_name'] = check_vol['name'] + check_binfo['release'] = draft_release + checkdir = koji.pathinfo.build(check_binfo) + if os.path.islink(checkdir): + os.unlink(checkdir) + + def reset_build(build): """Reset a build so that it can be reimported @@ -8916,7 +8947,7 @@ def reset_build(build): data={'state': binfo['state'], 'task_id': None, 'volume_id': 0}) update.execute() - _clean_draft_link(binfo) + _delete_build_symlinks(binfo) # now clear the build dir builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): @@ -16040,22 +16071,3 @@ def draft_clause(draft, table=None): else: # null is included return f'{table}draft IS NOT TRUE' - - -def _clean_draft_link(promoted_build): - """remove the symlink of old builddir to promoted builddir""" - draft_info = (promoted_build.get('extra') or {}).get('draft', {}) - if not (draft_info and draft_info.get('promoted')): - # skipped as it's not a promoted build. - return - old_release = draft_info.get('old_release') - if not old_release: - raise koji.GenericError( - f"extra.draft.old_release not found in build: {promoted_build['nvr']}" - ) - draft_buildinfo = dslice(promoted_build, ['name', 'version', 'volume_name']) - draft_buildinfo['release'] = old_release - draft_builddir = koji.pathinfo.build(draft_buildinfo) - # only unlink whe its a symlink. - if os.path.islink(draft_builddir): - os.unlink(draft_builddir) From fdbfb8a8823a1fcae92647637be20a8c2fff9e86 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:57 +0000 Subject: [PATCH 56/70] use default volume for draft symlink when promoting --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 192a056..36312d7 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -10323,9 +10323,21 @@ def _promote_build(build, force=False): update.execute() new_binfo = get_build(binfo['id'], strict=True) - move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) + oldpath = koji.pathinfo.build(binfo) + newpath = koji.pathinfo.build(new_binfo) + safer_move(oldpath, newpath) ensure_volume_symlink(new_binfo) + # provide a symlink at original draft location + # we point to the default volume in case the build moves in the future + base_vol = lookup_name('volume', 'DEFAULT', strict=True) + base_binfo = new_binfo.copy() + base_binfo['volume_id'] = base_vol['id'] + base_binfo['volume_name'] = base_vol['name'] + basedir = koji.pathinfo.build(base_binfo) + relpath = os.path.relpath(basedir, os.path.dirname(oldpath)) + os.symlink(relpath, oldpath) + # apply volume policy in case it's changed by release update. apply_volume_policy(new_binfo, strict=False) From 09fee79b3dda68471a6890bdb47e2d77d24f33b8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:57 +0000 Subject: [PATCH 57/70] block draft builds that can't be promoted --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 36312d7..62e6b61 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6116,12 +6116,10 @@ def new_build(data, strict=False): data.setdefault('volume_id', 0) data.setdefault('draft', False) - old_binfo = None - if not data.get('draft'): - # check for existing build - old_binfo = get_build(data) + # check for existing build + old_binfo = get_build(data) if old_binfo: - if strict: + if strict or data['draft']: raise koji.GenericError(f'Existing build found: {old_binfo}') recycle_build(old_binfo, data) # Raises exception if there is a problem From 4ba5ba0e5eddc0d309ba9d561a2fe41a4fd09940 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:58 +0000 Subject: [PATCH 58/70] add a comment in import_rpm --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 62e6b61..82f0a3c 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6414,6 +6414,8 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): if draft: # shouldn't happen with current code raise koji.GenericError('rpm import is not supported for draft builds') + # the existing build got by rpminfo below should never be draft, + # because "," is an invalid char for rpm "release" if rpminfo['sourcepackage'] == 1: buildinfo = get_build(rpminfo, strict=False) if not buildinfo: From 699709739227743ec9f97928688bb55aaf4b4f9f Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 05 2024 10:00:58 +0000 Subject: [PATCH 59/70] fix tests --- diff --git a/tests/test_hub/test_delete_build.py b/tests/test_hub/test_delete_build.py index 119a51c..9fd50ac 100644 --- a/tests/test_hub/test_delete_build.py +++ b/tests/test_hub/test_delete_build.py @@ -12,7 +12,6 @@ UP = kojihub.UpdateProcessor class TestDeleteBuild(unittest.TestCase): - def getDelete(self, *args, **kwargs): delete = DP(*args, **kwargs) delete.execute = mock.MagicMock() @@ -32,115 +31,140 @@ class TestDeleteBuild(unittest.TestCase): return update def setUp(self): - self.DeleteProcessor = mock.patch('kojihub.kojihub.DeleteProcessor', - side_effect=self.getDelete).start() + self.DeleteProcessor = mock.patch( + "kojihub.kojihub.DeleteProcessor", side_effect=self.getDelete + ).start() self.deletes = [] - self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor', - side_effect=self.getQuery).start() + self.QueryProcessor = mock.patch( + "kojihub.kojihub.QueryProcessor", side_effect=self.getQuery + ).start() self.queries = [] self.query_execute = mock.MagicMock() - self.UpdateProcessor = mock.patch('kojihub.kojihub.UpdateProcessor', - side_effect=self.getUpdate).start() + self.UpdateProcessor = mock.patch( + "kojihub.kojihub.UpdateProcessor", side_effect=self.getUpdate + ).start() self.updates = [] - self.context_db = mock.patch('kojihub.db.context').start() + self.context_db = mock.patch("kojihub.db.context").start() self.context_db.session.assertLogin = mock.MagicMock() self.context_db.event_id = 42 self.context_db.session.user_id = 24 - self.get_build = mock.patch('kojihub.kojihub.get_build').start() - self._delete_build = mock.patch('kojihub.kojihub._delete_build').start() - self.get_user = mock.patch('kojihub.kojihub.get_user').start() - self.context = mock.patch('kojihub.kojihub.context').start() + self.get_build = mock.patch("kojihub.kojihub.get_build").start() + self._delete_build = mock.patch("kojihub.kojihub._delete_build").start() + self.get_user = mock.patch("kojihub.kojihub.get_user").start() + self.context = mock.patch("kojihub.kojihub.context").start() self.context.session.assertPerm = mock.MagicMock() - self.binfo = {'id': 'BUILD ID', 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'test_nvr', - 'nvr': 'test_nvr-3.3-20.el8', 'version': '3.3', 'release': '20'} + self.binfo = { + "id": "BUILD ID", + "state": koji.BUILD_STATES["COMPLETE"], + "name": "test_nvr", + "nvr": "test_nvr-3.3-20.el8", + "version": "3.3", + "release": "20", + "volume_id": 1, + "volume_name": 'testvol', + "draft": False + } def tearDown(self): mock.patch.stopall() def test_delete_build_raise_error(self): - references = ['tags', 'rpms', 'archives', 'component_of'] + references = ["tags", "rpms", "archives", "component_of"] for ref in references: context = mock.MagicMock() context.session.return_value = context - with mock.patch('kojihub.kojihub.build_references') as refs: + with mock.patch("kojihub.kojihub.build_references") as refs: retval = defaultdict(dict) retval[ref] = True refs.return_value = retval with self.assertRaises(koji.GenericError): - kojihub.delete_build(build='', strict=True) + kojihub.delete_build(build="", strict=True) def test_delete_build_return_false(self): - references = ['tags', 'rpms', 'archives', 'component_of'] + references = ["tags", "rpms", "archives", "component_of"] for ref in references: context = mock.MagicMock() context.session.return_value = context - with mock.patch('kojihub.kojihub.build_references') as refs: + with mock.patch("kojihub.kojihub.build_references") as refs: retval = defaultdict(dict) retval[ref] = True refs.return_value = retval - assert kojihub.delete_build(build='', strict=False) is False + assert kojihub.delete_build(build="", strict=False) is False def test_delete_build_check_last_used_raise_error(self): - references = ['tags', 'rpms', 'archives', 'component_of', 'last_used'] + references = ["tags", "rpms", "archives", "component_of", "last_used"] for ref in references: context = mock.MagicMock() context.session.return_value = context - with mock.patch('kojihub.kojihub.build_references') as refs: + with mock.patch("kojihub.kojihub.build_references") as refs: retval = defaultdict(dict) - if ref == 'last_used': + if ref == "last_used": retval[ref] = time.time() + 100 refs.return_value = retval - self.assertFalse(kojihub.delete_build(build='', strict=False)) + self.assertFalse(kojihub.delete_build(build="", strict=False)) - @mock.patch('kojihub.kojihub.build_references') + @mock.patch("kojihub.kojihub.build_references") def test_delete_build_lazy_refs(self, buildrefs): - '''Test that we can handle lazy return from build_references''' - self.get_user.return_value = {'authtype': 2, 'id': 1, 'krb_principal': None, - 'krb_principals': [], 'name': 'kojiadmin', 'status': 0, - 'usertype': 0} - buildrefs.return_value = {'tags': []} + """Test that we can handle lazy return from build_references""" + self.get_user.return_value = { + "authtype": 2, + "id": 1, + "krb_principal": None, + "krb_principals": [], + "name": "kojiadmin", + "status": 0, + "usertype": 0, + } + buildrefs.return_value = {"tags": []} self.get_build.return_value = self.binfo kojihub.delete_build(build=self.binfo, strict=True) # no build refs, so we should have called _delete_build self._delete_build.assert_called_with(self.binfo) - def test_delete_build_queries(self): - self.query_execute.return_value = [(123, )] + @mock.patch("os.unlink") + @mock.patch("koji.util.rmtree") + def test_delete_build_queries(self, rmtree, unlink): + self.query_execute.side_effect = [ + [(123,)], # rpm ids + {'id': 0, 'name': 'DEFAULT'}, # volume DEFAULT + [{'id': 0, 'name': 'DEFAULT'}, + {'id': 1, 'name': 'testvol'}, + {'id': 2, 'name': 'other'}] # list_volumes() + ] kojihub._delete_build(self.binfo) - self.assertEqual(len(self.queries), 1) + self.assertEqual(len(self.queries), 3) query = self.queries[0] - self.assertEqual(query.tables, ['rpminfo']) + self.assertEqual(query.tables, ["rpminfo"]) self.assertEqual(query.joins, None) - self.assertEqual(query.clauses, ['build_id=%(build_id)i']) - self.assertEqual(query.columns, ['id']) + self.assertEqual(query.clauses, ["build_id=%(build_id)i"]) + self.assertEqual(query.columns, ["id"]) self.assertEqual(len(self.deletes), 2) delete = self.deletes[0] - self.assertEqual(delete.table, 'rpmsigs') + self.assertEqual(delete.table, "rpmsigs") self.assertEqual(delete.clauses, ["rpm_id=%(rpm_id)i"]) delete = self.deletes[1] - self.assertEqual(delete.table, 'rpm_checksum') + self.assertEqual(delete.table, "rpm_checksum") self.assertEqual(delete.clauses, ["rpm_id=%(rpm_id)i"]) self.assertEqual(len(self.updates), 2) update = self.updates[0] - self.assertEqual(update.table, 'tag_listing') - self.assertEqual(update.values, {'build_id': self.binfo['id']}) - self.assertEqual(update.data, {'revoke_event': 42, 'revoker_id': 24}) - self.assertEqual(update.rawdata, {'active': 'NULL'}) - self.assertEqual(update.clauses, ["build_id=%(build_id)i", 'active = TRUE']) + self.assertEqual(update.table, "tag_listing") + self.assertEqual(update.values, {"build_id": self.binfo["id"]}) + self.assertEqual(update.data, {"revoke_event": 42, "revoker_id": 24}) + self.assertEqual(update.rawdata, {"active": "NULL"}) + self.assertEqual(update.clauses, ["build_id=%(build_id)i", "active = TRUE"]) update = self.updates[1] - self.assertEqual(update.table, 'build') - self.assertEqual(update.values, {'build_id': self.binfo['id']}) - self.assertEqual(update.data, {'state': 2}) + self.assertEqual(update.table, "build") + self.assertEqual(update.values, {"build_id": self.binfo["id"]}) + self.assertEqual(update.data, {"state": 2}) self.assertEqual(update.rawdata, {}) - self.assertEqual(update.clauses, ['id=%(build_id)i']) - + self.assertEqual(update.clauses, ["id=%(build_id)i"]) diff --git a/tests/test_hub/test_import_rpm.py b/tests/test_hub/test_import_rpm.py index 314dc62..cfc3326 100644 --- a/tests/test_hub/test_import_rpm.py +++ b/tests/test_hub/test_import_rpm.py @@ -211,23 +211,6 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(insert.data, data) self.assertEqual(insert.rawdata, {}) - def test_import_draft_conflict(self): - with self.assertRaises(koji.GenericError) as cm: - kojihub.import_rpm(self.filename, buildinfo={'id': 1024, 'draft': False}, draft=True) - self.assertEqual("draft property: False of build: 1024 mismatch, True is expected", - str(cm.exception)) - self.assertEqual(len(self.inserts), 0) - - def test_import_draft_rpm_without_buildinfo(self): - self.os_path_basename.return_value = 'name-version-release.arch.rpm' - self.get_rpm_header.return_value = self.rpm_header_retval - - with self.assertRaises(koji.GenericError) as cm: - kojihub.import_rpm(self.filename, draft=True) - self.assertEqual(f"Cannot import draft rpm: {self.os_path_basename.return_value}" - " without specifying a build", str(cm.exception)) - self.assertEqual(len(self.inserts), 0) - def test_import_draft_rpm_invalid_release(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' self.get_rpm_header.return_value = self.rpm_header_retval @@ -242,7 +225,7 @@ class TestImportRPM(unittest.TestCase): } with self.assertRaises(koji.GenericError) as cm: - kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) + kojihub.import_rpm(self.filename, buildinfo=buildinfo) self.assertEqual( 'draft release: badrelease is not in valid format', str(cm.exception) @@ -267,7 +250,7 @@ class TestImportRPM(unittest.TestCase): } } self.nextval.return_value = 9876 - kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) + kojihub.import_rpm(self.filename, buildinfo=buildinfo) data = { 'build_id': 12345, 'name': 'name', @@ -308,7 +291,7 @@ class TestImportRPM(unittest.TestCase): 'draft': True } self.nextval.return_value = 9876 - kojihub.import_rpm(self.src_filename, buildinfo=buildinfo, draft=True) + kojihub.import_rpm(self.src_filename, buildinfo=buildinfo) data = { 'build_id': 12345, 'name': 'name', @@ -329,52 +312,3 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(insert.table, 'rpminfo') self.assertEqual(insert.data, data) self.assertEqual(insert.rawdata, {}) - - def test_import_draft_srpm_without_buildinfo(self): - self.os_path_basename.return_value = 'name-version-release.src.rpm' - retval = copy.copy(self.rpm_header_retval) - retval.update({ - 'filename': 'name-version-release.src.rpm', - 1044: 'name-version-release.src.rpm.bad', - 1022: 'src', - 1106: 1, - }) - self.get_rpm_header.return_value = retval - self.get_build.return_value = { - 'state': koji.BUILD_STATES['COMPLETE'], - 'name': 'name', - 'version': 'version', - 'release': 'release', - 'id': 5566, - 'draft': True, - 'extra': { - 'draft': { - 'target_release': 'release' - } - } - } - self.new_build.return_value = 5566 - self.nextval.return_value = 9876 - kojihub.import_rpm(self.src_filename, draft=True) - data = { - 'build_id': 5566, - 'name': 'name', - 'arch': 'src', - 'buildtime': 'buildtime', - 'draft': True, - 'payloadhash': '7061796c6f61642068617368', - 'epoch': 'epoch', - 'version': 'version', - 'buildroot_id': None, - 'release': 'release', - 'external_repo_id': 0, - 'id': 9876, - 'size': 0, - } - self.assertEqual(len(self.inserts), 1) - insert = self.inserts[0] - self.assertEqual(insert.table, 'rpminfo') - self.assertEqual(insert.data, data) - self.assertEqual(insert.rawdata, {}) - self.get_build.assert_called_once_with(5566, strict=True) - self.assertEqual(self.get_build.call_count, 1) diff --git a/tests/test_hub/test_new_build.py b/tests/test_hub/test_new_build.py index 9e7bd74..758e0da 100644 --- a/tests/test_hub/test_new_build.py +++ b/tests/test_hub/test_new_build.py @@ -2,7 +2,6 @@ import mock import unittest import koji -from koji.util import dslice import kojihub IP = kojihub.InsertProcessor @@ -10,21 +9,24 @@ IP = kojihub.InsertProcessor class TestNewBuild(unittest.TestCase): def setUp(self): - self.get_rpm = mock.patch('kojihub.kojihub.get_rpm').start() - self.get_external_repo_id = mock.patch('kojihub.kojihub.get_external_repo_id').start() - self.nextval = mock.patch('kojihub.kojihub.nextval').start() - self.Savepoint = mock.patch('kojihub.kojihub.Savepoint').start() - self.InsertProcessor = mock.patch('kojihub.kojihub.InsertProcessor', - side_effect=self.getInsert).start() + self.get_rpm = mock.patch("kojihub.kojihub.get_rpm").start() + self.get_external_repo_id = mock.patch( + "kojihub.kojihub.get_external_repo_id" + ).start() + self.nextval = mock.patch("kojihub.kojihub.nextval").start() + self.Savepoint = mock.patch("kojihub.kojihub.Savepoint").start() + self.InsertProcessor = mock.patch( + "kojihub.kojihub.InsertProcessor", side_effect=self.getInsert + ).start() self.inserts = [] self.insert_execute = mock.MagicMock() - self.lookup_package = mock.patch('kojihub.kojihub.lookup_package').start() - self.new_package = mock.patch('kojihub.kojihub.new_package').start() - self.get_user = mock.patch('kojihub.kojihub.get_user').start() - self.get_build = mock.patch('kojihub.kojihub.get_build').start() - self.recycle_build = mock.patch('kojihub.kojihub.recycle_build').start() - self.context = mock.patch('kojihub.kojihub.context').start() - self.find_build_id = mock.patch('kojihub.kojihub.find_build_id').start() + self.lookup_package = mock.patch("kojihub.kojihub.lookup_package").start() + self.new_package = mock.patch("kojihub.kojihub.new_package").start() + self.get_user = mock.patch("kojihub.kojihub.get_user").start() + self.get_build = mock.patch("kojihub.kojihub.get_build").start() + self.recycle_build = mock.patch("kojihub.kojihub.recycle_build").start() + self.context = mock.patch("kojihub.kojihub.context").start() + self.find_build_id = mock.patch("kojihub.kojihub.find_build_id").start() def tearDown(self): mock.patch.stopall() @@ -39,37 +41,40 @@ class TestNewBuild(unittest.TestCase): self.get_build.return_value = None self.nextval.return_value = 65 # free build id self.new_package.return_value = 54 - self.get_user.return_value = {'id': 123} + self.get_user.return_value = {"id": 123} data = { - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', - 'owner': 'test_owner', - 'extra': {'extra_key': 'extra_value'}, + "name": "test_name", + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", + "owner": "test_owner", + "extra": {"extra_key": "extra_value"}, } kojihub.new_build(data) self.assertEqual(len(self.inserts), 1) insert = self.inserts[0] - self.assertEqual(insert.table, 'build') - self.assertEqual(insert.data, { - 'completion_time': 'NOW', - 'epoch': 'test_epoch', - 'extra': '{"extra_key": "extra_value"}', - 'id': 65, - 'owner': 123, - 'pkg_id': 54, - 'release': 'test_release', - 'source': None, - 'start_time': 'NOW', - 'state': 1, - 'task_id': None, - 'draft': False, - 'version': 'test_version', - 'volume_id': 0 - }) + self.assertEqual(insert.table, "build") + self.assertEqual( + insert.data, + { + "completion_time": "NOW", + "epoch": "test_epoch", + "extra": '{"extra_key": "extra_value"}', + "id": 65, + "owner": 123, + "pkg_id": 54, + "release": "test_release", + "source": None, + "start_time": "NOW", + "state": 1, + "task_id": None, + "draft": False, + "version": "test_version", + "volume_id": 0, + }, + ) def test_empty_data(self): with self.assertRaises(koji.GenericError): @@ -79,13 +84,13 @@ class TestNewBuild(unittest.TestCase): def test_wrong_pkg_id(self): self.lookup_package.side_effect = koji.GenericError data = { - 'pkg_id': 444, - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', - 'owner': 'test_owner', - 'extra': {'extra_key': 'extra_value'}, + "pkg_id": 444, + "name": "test_name", + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", + "owner": "test_owner", + "extra": {"extra_key": "extra_value"}, } with self.assertRaises(koji.GenericError): @@ -95,11 +100,11 @@ class TestNewBuild(unittest.TestCase): def test_missing_pkg_id_name(self): data = { - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', - 'owner': 'test_owner', - 'extra': {'extra_key': 'extra_value'}, + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", + "owner": "test_owner", + "extra": {"extra_key": "extra_value"}, } with self.assertRaises(koji.GenericError) as cm: @@ -111,12 +116,12 @@ class TestNewBuild(unittest.TestCase): def test_wrong_owner(self): self.get_user.side_effect = koji.GenericError data = { - 'owner': 123456, - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', - 'extra': {'extra_key': 'extra_value'}, + "owner": 123456, + "name": "test_name", + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", + "extra": {"extra_key": "extra_value"}, } with self.assertRaises(koji.GenericError): @@ -126,13 +131,13 @@ class TestNewBuild(unittest.TestCase): def test_missing_vre(self): data = { - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', + "name": "test_name", + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", } - for item in ('version', 'release', 'epoch'): + for item in ("version", "release", "epoch"): d = data.copy() del d[item] with self.assertRaises(koji.GenericError): @@ -146,61 +151,88 @@ class TestNewBuild(unittest.TestCase): pass data = { - 'owner': 123456, - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', - 'extra': {'extra_key': CantDoJSON()}, + "owner": 123456, + "name": "test_name", + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", + "extra": {"extra_key": CantDoJSON()}, } with self.assertRaises(koji.GenericError) as cm: kojihub.new_build(data) self.assertEqual(len(self.inserts), 0) - self.assertEqual("No such build extra data: %(extra)r" % data, str(cm.exception)) + self.assertEqual( + "No such build extra data: %(extra)r" % data, str(cm.exception) + ) def test_draft(self): data = { - 'owner': 123456, - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release', - 'epoch': 'test_epoch', - 'draft': True + "owner": 123456, + "name": "test_name", + "version": "test_version", + "release": "test_release", + "epoch": "test_epoch", + "draft": True, } insert_data = { - 'completion_time': 'NOW', - 'epoch': 'test_epoch', - 'extra': '{"draft": {"target_release": "test_release", "promoted": false}}', - 'id': 108, - 'owner': 123, - 'pkg_id': 54, - 'release': 'test_release,draft_108', - 'source': None, - 'start_time': 'NOW', - 'state': 1, - 'task_id': None, - 'draft': True, - 'version': 'test_version', - 'volume_id': 0 + "completion_time": "NOW", + "epoch": "test_epoch", + "extra": '{"draft": {"target_release": "test_release", "promoted": false}}', + "id": 108, + "owner": 123, + "pkg_id": 54, + "release": "test_release,draft_108", + "source": None, + "start_time": "NOW", + "state": 1, + "task_id": None, + "draft": True, + "version": "test_version", + "volume_id": 0, } self.nextval.return_value = 108 self.new_package.return_value = 54 - self.get_user.return_value = {'id': 123} + self.get_user.return_value = {"id": 123} + self.get_build.side_effect = [None, mock.ANY] self.find_build_id.return_value = None kojihub.new_build(data) self.assertEqual(len(self.inserts), 1) insert = self.inserts[0] - self.assertEqual(insert.table, 'build') + self.assertEqual(insert.table, "build") self.assertEqual(insert.data, insert_data) - self.get_build.assert_called_once_with(108, strict=True) - self.assertEqual(self.get_build.call_count, 1) + self.get_build.assert_has_calls( + [ + mock.call( + # it looks like a "draft build" because we use the "data" reference + { + "owner": 123, + "name": "test_name", + "version": "test_version", + "release": "test_release,draft_108", + "epoch": "test_epoch", + "draft": True, + "pkg_id": 54, + "extra": None, + "state": 1, + "start_time": "NOW", + "completion_time": "NOW", + "source": None, + "task_id": None, + "volume_id": 0, + "id": 108, + } + ), + mock.call(108, strict=True), + ] + ) self.find_build_id.assert_called_once_with( { - 'name': 'test_name', - 'version': 'test_version', - 'release': 'test_release,draft_108' - }) + "name": "test_name", + "version": "test_version", + "release": "test_release,draft_108", + } + ) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index a3b05f3..b554e66 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -31,8 +31,11 @@ class TestPromoteBuild(unittest.TestCase): self.assert_policy = mock.patch('kojihub.kojihub.assert_policy').start() self.apply_volume_policy = mock.patch('kojihub.kojihub.apply_volume_policy', return_value=None).start() - self.move_and_symlink = mock.patch('kojihub.kojihub.move_and_symlink').start() + self.safer_move = mock.patch('kojihub.kojihub.safer_move').start() self.ensure_volume_symlink = mock.patch('kojihub.kojihub.ensure_volume_symlink').start() + self.lookup_name = mock.patch('kojihub.kojihub.lookup_name', + return_value={'id': 1, 'name': 'DEFAULT'}).start() + self.os_symlink = mock.patch('os.symlink').start() self.list_tags = mock.patch('kojihub.kojihub.list_tags', return_value=[{'id': 101}]).start() self.set_tag_update = mock.patch('kojihub.kojihub.set_tag_update').start() @@ -107,6 +110,14 @@ class TestPromoteBuild(unittest.TestCase): self.apply_volume_policy.assert_called_once_with( self.new_build, strict=False ) + self.safer_move.assert_called_once_with( + '/mnt/koji/vol/X/packages/foo/bar/tgtrel,draft_1', + '/mnt/koji/vol/X/packages/foo/bar/tgtrel' + ) + self.os_symlink.assert_called_once_with( + '../../../../../packages/foo/bar/tgtrel', + '/mnt/koji/vol/X/packages/foo/bar/tgtrel,draft_1' + ) def test_promote_build_not_draft(self): self.get_build.return_value = {'draft': False, 'nvr': 'testnvr'} diff --git a/tests/test_hub/test_reset_build.py b/tests/test_hub/test_reset_build.py index 06fe49d..23a79d9 100644 --- a/tests/test_hub/test_reset_build.py +++ b/tests/test_hub/test_reset_build.py @@ -10,7 +10,6 @@ UP = kojihub.UpdateProcessor class TestResetBuild(unittest.TestCase): - def getDelete(self, *args, **kwargs): delete = DP(*args, **kwargs) delete.execute = mock.MagicMock() @@ -31,29 +30,49 @@ class TestResetBuild(unittest.TestCase): def setUp(self): self.maxDiff = None - self.DeleteProcessor = mock.patch('kojihub.kojihub.DeleteProcessor', - side_effect=self.getDelete).start() + self.DeleteProcessor = mock.patch( + "kojihub.kojihub.DeleteProcessor", side_effect=self.getDelete + ).start() self.deletes = [] - self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor', - side_effect=self.getQuery).start() + self.QueryProcessor = mock.patch( + "kojihub.kojihub.QueryProcessor", side_effect=self.getQuery + ).start() self.queries = [] self.query_execute = mock.MagicMock() - self.UpdateProcessor = mock.patch('kojihub.kojihub.UpdateProcessor', - side_effect=self.getUpdate).start() + self.UpdateProcessor = mock.patch( + "kojihub.kojihub.UpdateProcessor", side_effect=self.getUpdate + ).start() self.updates = [] - self.get_build = mock.patch('kojihub.kojihub.get_build').start() - self.context = mock.patch('kojihub.kojihub.context').start() + self.get_build = mock.patch("kojihub.kojihub.get_build").start() + self.context = mock.patch("kojihub.kojihub.context").start() self.context.session.assertPerm = mock.MagicMock() # don't remove anything unexpected - self.rmtree = mock.patch('koji.util.rmtree').start() - self.unlink = mock.patch('os.unlink').start() + self.rmtree = mock.patch("koji.util.rmtree").start() + self.unlink = mock.patch("os.unlink").start() self.build_id = 3 - self.binfo = {'id': 3, 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'test_nvr', - 'nvr': 'test_nvr-3.3-20.el8', 'version': '3.3', 'release': '20', - 'task_id': 12, 'volume_id': 1, 'build_id': 3} - self.del_binfo = {'id': 3, 'state': koji.BUILD_STATES['CANCELED'], - 'name': 'test_nvr', 'nvr': 'test_nvr-3.3-20.el8', 'version': '3.3', - 'release': '20', 'task_id': None, 'volume_id': 0} + self.binfo = { + "id": 3, + "state": koji.BUILD_STATES["COMPLETE"], + "name": "test_nvr", + "nvr": "test_nvr-3.3-20.el8", + "version": "3.3", + "release": "20", + "task_id": 12, + "volume_id": 1, + "build_id": 3, + "draft": False + } + self.del_binfo = { + "id": 3, + "state": koji.BUILD_STATES["CANCELED"], + "name": "test_nvr", + "nvr": "test_nvr-3.3-20.el8", + "version": "3.3", + "release": "20", + "task_id": None, + "volume_id": 0, + "draft": False + } def tearDown(self): mock.patch.stopall() @@ -61,128 +80,135 @@ class TestResetBuild(unittest.TestCase): def test_reset_build_queries(self): self.get_build.side_effect = [self.binfo, self.del_binfo] self.query_execute.side_effect = [ - [(123, )], - [(9999,)], + [(123,)], # rpm ids + [(9999,)], # archive ids + {"id": 0, "name": "DEFAULT"}, # volume DEFAULT + [ + {"id": 0, "name": "DEFAULT"}, + {"id": 1, "name": "testvol"}, + {"id": 2, "name": "other"}, + ], # list_volumes() ] kojihub.reset_build(self.build_id) - self.assertEqual(len(self.queries), 2) + self.assertEqual(len(self.queries), 4) query = self.queries[0] - self.assertEqual(query.tables, ['rpminfo']) + self.assertEqual(query.tables, ["rpminfo"]) self.assertEqual(query.joins, None) - self.assertEqual(query.clauses, ['build_id=%(id)i']) - self.assertEqual(query.columns, ['id']) - self.assertEqual(query.values, {'id': self.binfo['build_id']}) + self.assertEqual(query.clauses, ["build_id=%(id)i"]) + self.assertEqual(query.columns, ["id"]) + self.assertEqual(query.values, {"id": self.binfo["build_id"]}) query = self.queries[1] - self.assertEqual(query.tables, ['archiveinfo']) + self.assertEqual(query.tables, ["archiveinfo"]) self.assertEqual(query.joins, None) - self.assertEqual(query.clauses, ['build_id=%(id)i']) - self.assertEqual(query.columns, ['id']) - self.assertEqual(query.values, {'id': self.binfo['build_id']}) + self.assertEqual(query.clauses, ["build_id=%(id)i"]) + self.assertEqual(query.columns, ["id"]) + self.assertEqual(query.values, {"id": self.binfo["build_id"]}) self.assertEqual(len(self.updates), 1) update = self.updates[0] - self.assertEqual(update.table, 'build') - self.assertEqual(update.values, {'id': self.binfo['id']}) - self.assertEqual(update.data, {'state': 4, 'task_id': None, 'volume_id': 0}) + self.assertEqual(update.table, "build") + self.assertEqual(update.values, {"id": self.binfo["id"]}) + self.assertEqual(update.data, {"state": 4, "task_id": None, "volume_id": 0}) self.assertEqual(update.rawdata, {}) - self.assertEqual(update.clauses, ['id=%(id)s']) + self.assertEqual(update.clauses, ["id=%(id)s"]) self.assertEqual(len(self.deletes), 18) delete = self.deletes[0] - self.assertEqual(delete.table, 'rpmsigs') - self.assertEqual(delete.clauses, ['rpm_id=%(rpm_id)i']) - self.assertEqual(delete.values, {'rpm_id': 123}) + self.assertEqual(delete.table, "rpmsigs") + self.assertEqual(delete.clauses, ["rpm_id=%(rpm_id)i"]) + self.assertEqual(delete.values, {"rpm_id": 123}) delete = self.deletes[1] - self.assertEqual(delete.table, 'buildroot_listing') - self.assertEqual(delete.clauses, ['rpm_id=%(rpm_id)i']) - self.assertEqual(delete.values, {'rpm_id': 123}) + self.assertEqual(delete.table, "buildroot_listing") + self.assertEqual(delete.clauses, ["rpm_id=%(rpm_id)i"]) + self.assertEqual(delete.values, {"rpm_id": 123}) delete = self.deletes[2] - self.assertEqual(delete.table, 'archive_rpm_components') - self.assertEqual(delete.clauses, ['rpm_id=%(rpm_id)i']) - self.assertEqual(delete.values, {'rpm_id': 123}) + self.assertEqual(delete.table, "archive_rpm_components") + self.assertEqual(delete.clauses, ["rpm_id=%(rpm_id)i"]) + self.assertEqual(delete.values, {"rpm_id": 123}) delete = self.deletes[3] - self.assertEqual(delete.table, 'rpm_checksum') - self.assertEqual(delete.clauses, ['rpm_id=%(rpm_id)i']) - self.assertEqual(delete.values, {'rpm_id': 123}) + self.assertEqual(delete.table, "rpm_checksum") + self.assertEqual(delete.clauses, ["rpm_id=%(rpm_id)i"]) + self.assertEqual(delete.values, {"rpm_id": 123}) delete = self.deletes[4] - self.assertEqual(delete.table, 'rpminfo') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "rpminfo") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) delete = self.deletes[5] - self.assertEqual(delete.table, 'maven_archives') - self.assertEqual(delete.clauses, ['archive_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "maven_archives") + self.assertEqual(delete.clauses, ["archive_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[6] - self.assertEqual(delete.table, 'win_archives') - self.assertEqual(delete.clauses, ['archive_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "win_archives") + self.assertEqual(delete.clauses, ["archive_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[7] - self.assertEqual(delete.table, 'image_archives') - self.assertEqual(delete.clauses, ['archive_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "image_archives") + self.assertEqual(delete.clauses, ["archive_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[8] - self.assertEqual(delete.table, 'buildroot_archives') - self.assertEqual(delete.clauses, ['archive_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "buildroot_archives") + self.assertEqual(delete.clauses, ["archive_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[9] - self.assertEqual(delete.table, 'archive_rpm_components') - self.assertEqual(delete.clauses, ['archive_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "archive_rpm_components") + self.assertEqual(delete.clauses, ["archive_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[10] - self.assertEqual(delete.table, 'archive_components') - self.assertEqual(delete.clauses, ['archive_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "archive_components") + self.assertEqual(delete.clauses, ["archive_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[11] - self.assertEqual(delete.table, 'archive_components') - self.assertEqual(delete.clauses, ['component_id=%(archive_id)i']) - self.assertEqual(delete.values, {'archive_id': 9999}) + self.assertEqual(delete.table, "archive_components") + self.assertEqual(delete.clauses, ["component_id=%(archive_id)i"]) + self.assertEqual(delete.values, {"archive_id": 9999}) delete = self.deletes[12] - self.assertEqual(delete.table, 'archiveinfo') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "archiveinfo") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) delete = self.deletes[13] - self.assertEqual(delete.table, 'maven_builds') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "maven_builds") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) delete = self.deletes[14] - self.assertEqual(delete.table, 'win_builds') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "win_builds") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) delete = self.deletes[15] - self.assertEqual(delete.table, 'image_builds') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "image_builds") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) delete = self.deletes[16] - self.assertEqual(delete.table, 'build_types') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "build_types") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) delete = self.deletes[17] - self.assertEqual(delete.table, 'tag_listing') - self.assertEqual(delete.clauses, ['build_id=%(id)i']) - self.assertEqual(delete.values, {'id': self.binfo['build_id']}) + self.assertEqual(delete.table, "tag_listing") + self.assertEqual(delete.clauses, ["build_id=%(id)i"]) + self.assertEqual(delete.values, {"id": self.binfo["build_id"]}) - self.get_build.assert_has_calls([mock.call(self.build_id), - mock.call(self.build_id, strict=True)]) + self.get_build.assert_has_calls( + [mock.call(self.build_id), mock.call(self.build_id, strict=True)] + ) def test_reset_build_non_exist_build(self): self.get_build.return_value = None From 55917097354ecd71e47f1eea2425ea0200a406fe Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:59 +0000 Subject: [PATCH 60/70] docstring updates --- diff --git a/builder/kojid b/builder/kojid index eb31a51..8b85b9e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -895,31 +895,18 @@ class BuildRoot(object): def mapInternalRPMs(self, rpmlist): """ - Map each rpm item of rpmlist by attaching rpm_id in repodir's rpmlist.jsonl which - is generated while repo initiation. - - Skipped if rpmlist.jsonl doesn't exist for backward compatibility. - - For now, rpmlist.jsonl is a dict[nvra, rpmdata], where rpmdata contains keys below: - - id - - build_id - - name - - version - - release - - epoch - - arch - - payloadhash - - size - - buildtime - - Notes: - - Because the nvra uniqueness is still being kept, we don't need the check so far. - - it must be called after markExternalRPMs, as it use "external_repo" to skip external - rpms. + Map each rpm item of rpmlist to a specific koji rpm entry based on repo contents + + The rpmList should be a list of dicts containing rpm header values. These entries will be + modified in place to include an id field when mapped. + + This mapping relies on the rpmlist.jsonl file for the repo. If this file is missing, the + code will fall back to querying the hub. + + This function will raise an error if there is a sigmd5 mismatch for a given rpm. :param list rpmlist: rpm list fetched from local RPMDB. - :return: the same rpmlist as the parameter with build nvrs attached - :rtype: list + :return: None """ opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) From a4b4aedba8748de012e3c701ef54e3c8936d4871 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:59 +0000 Subject: [PATCH 61/70] fix compat_mode case --- diff --git a/builder/kojid b/builder/kojid index 8b85b9e..fe35e47 100755 --- a/builder/kojid +++ b/builder/kojid @@ -911,14 +911,10 @@ class BuildRoot(object): opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) rpmlist_path = os.path.join(self.get_repo_dir(), self.br_arch, 'rpmlist.jsonl') - repo_rpms = {} compat_mode = False try: with koji.openRemoteFile(rpmlist_path, **opts) as fo: - for line in fo: - repo_rpm = json.loads(line) - nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % repo_rpm - repo_rpms[nvra] = repo_rpm + repo_rpms = [json.loads(line) for line in fo] except requests.exceptions.HTTPError as e: if e.response.status_code == 404: self.logger.warning("Missing repo content file: %s", rpmlist_path) @@ -928,6 +924,7 @@ class BuildRoot(object): else: raise fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" + repo_rpms = {fmt % r: r for r in repo_rpms} for rpm_info in rpmlist: if 'external_repo' in rpm_info: continue From 71bead595502b96439d627b2a48fc17d40458e36 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:00:59 +0000 Subject: [PATCH 62/70] make error a bit clearer --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 82f0a3c..c2d7105 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6119,8 +6119,11 @@ def new_build(data, strict=False): # check for existing build old_binfo = get_build(data) if old_binfo: - if strict or data['draft']: - raise koji.GenericError(f'Existing build found: {old_binfo}') + old_str = '%(nvr)s (id=%(id)s)' % old_binfo + if data['draft']: + raise koji.GenericError(f'Target build already exists: {old_str}') + elif strict: + raise koji.GenericError(f'Existing build found: {old_str}') recycle_build(old_binfo, data) # Raises exception if there is a problem return old_binfo['id'] From 6e31a6c663d5e31a56f235eef939ed8f30b43382 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:00 +0000 Subject: [PATCH 63/70] avoid keyword-only args for now untagged_builds is exported and this aspect of the signature cannot be correctly reported by list-api --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index c2d7105..45ba3ff 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -1395,7 +1395,7 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None, pattern=None def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, - type=None, extra=False, *, draft=None): + type=None, extra=False, draft=None): """Returns a list of builds for specified tag :param int tag: tag ID @@ -8514,7 +8514,7 @@ def query_history(tables=None, **kwargs): return ret -def untagged_builds(name=None, queryOpts=None, *, draft=None): +def untagged_builds(name=None, queryOpts=None, draft=None): """Returns the list of untagged builds""" st_complete = koji.BUILD_STATES['COMPLETE'] # following can be achieved with simple query but with From 023f68d0c3582d0427bc70e736fe5cf8c81dafe9 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:01 +0000 Subject: [PATCH 64/70] avoid using build.extra for draft data --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 45ba3ff..036e458 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -68,7 +68,6 @@ from koji.util import ( base64encode, decode_bytes, dslice, - encode_datetime, extract_build_task, joinpath, md5_constructor, @@ -1435,10 +1434,12 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, ('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.promotion_time', 'promotion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), ('build.draft', 'draft'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), + ('promoter.id', 'promoter_id'), ('promoter.name', 'promoter_name'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ('package.id', 'package_id'), ('package.name', 'package_name'), @@ -1452,6 +1453,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, 'package ON package.id = build.pkg_id', 'volume ON volume.id = build.volume_id', 'users ON users.id = build.owner', + 'LEFT JOIN users AS promoter ON promoter.id = build.promoter', ] if type is None: @@ -4426,6 +4428,7 @@ def get_build(buildInfo, strict=False): ('build.draft', 'draft'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.promotion_time', 'promotion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), @@ -4436,7 +4439,9 @@ def get_build(buildInfo, strict=False): ("date_part('epoch', events.time)", 'creation_ts'), ("date_part('epoch', build.start_time)", 'start_ts'), ("date_part('epoch', build.completion_time)", 'completion_ts'), + ("date_part('epoch', build.promotion_time)", 'promotion_ts'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), + ('promoter.id', 'promoter_id'), ('promoter.name', 'promoter_name'), ('build.cg_id', 'cg_id'), ('build.source', 'source'), ('build.extra', 'extra')) @@ -4445,6 +4450,7 @@ def get_build(buildInfo, strict=False): 'package on build.pkg_id = package.id', 'volume on build.volume_id = volume.id', 'users on build.owner = users.id', + 'LEFT JOIN users AS promoter ON build.promoter = promoter.id', ] clauses = ['build.id = %(buildID)i'] query = QueryProcessor(columns=fields, aliases=aliases, values=locals(), @@ -6149,14 +6155,6 @@ def new_build(data, strict=False): raise koji.GenericError( f"The build already exists: {draft_nvr}" ) - # data.extra.draft should not contains anything - if extra.get('draft'): - raise koji.GenericError('"draft" found in input extra which is reserved already') - extra['draft'] = { - 'target_release': target_release, - 'promoted': False - } - insert_data['extra'] = json.dumps(extra) or None insert = InsertProcessor('build', data=insert_data) insert.execute() new_binfo = get_build(data['id'], strict=True) @@ -10309,20 +10307,9 @@ def _promote_build(build, force=False): user=user ) - # temp solution with python generated time, maybe better to be an event? - now = datetime.datetime.now() - - extra = binfo.get('extra') or {} - draft_info = extra.setdefault('draft', {}) - draft_info['promoted'] = True - draft_info['old_release'] = old_release - draft_info['promotion_time'] = encode_datetime(now) - draft_info['promotion_ts'] = now.timestamp() - draft_info['promoter'] = user['name'] - extra = json.dumps(extra) - update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) - update.set(draft=False, release=target_release, extra=extra) + update.set(draft=False, release=target_release, promoter=user['id']) + update.rawset(promotion_time='now()') update.execute() new_binfo = get_build(binfo['id'], strict=True) @@ -12184,6 +12171,8 @@ class RootExports(object): - nvr (synthesized for sorting purposes) - owner_id - owner_name + - promoter_id + - promoter_name - volume_id - volume_name - source @@ -12194,6 +12183,8 @@ class RootExports(object): - start_ts - completion_time - completion_ts + - promotion_time + - promotion_ts - task_id - extra @@ -12215,6 +12206,7 @@ class RootExports(object): ('build.draft', 'draft'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.promotion_time', 'promotion_time'), ('build.start_time', 'start_time'), ('build.source', 'source'), ('build.extra', 'extra'), @@ -12223,17 +12215,22 @@ class RootExports(object): ("date_part('epoch', events.time)", 'creation_ts'), ("date_part('epoch', build.start_time)", 'start_ts'), ("date_part('epoch', build.completion_time)", 'completion_ts'), + ("date_part('epoch', build.promotion_time)", 'promotion_ts'), ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), - ('users.id', 'owner_id'), ('users.name', 'owner_name')] + ('users.id', 'owner_id'), ('users.name', 'owner_name'), + ('promoter.id', 'promoter_id'), ('promoter.name', 'promoter_name'), + ] tables = ['build'] joins = ['LEFT JOIN events ON build.create_event = events.id', 'LEFT JOIN package ON build.pkg_id = package.id', 'LEFT JOIN volume ON build.volume_id = volume.id', - 'LEFT JOIN users ON build.owner = users.id'] + 'LEFT JOIN users ON build.owner = users.id', + 'LEFT JOIN users AS promoter ON build.promoter = promoter.id', + ] clauses = [] if packageID is not None: packageID = get_package_id(packageID) diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index 19642a2..8437b5e 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -54,8 +54,12 @@ BEGIN; INSERT INTO permissions (name, description) VALUES ('draft-promoter', 'The permission required in the default "draft_promotion" hub policy rule to promote draft build.'); ALTER TABLE build ADD COLUMN draft BOOLEAN NOT NULL DEFAULT 'false'; + ALTER TABLE build ADD COLUMN promotion_time TIMESTAMPTZ; + ALTER TABLE build ADD COLUMN promoter INTEGER; + ALTER TABLE build ADD CONSTRAINT build_promoter_fkey FOREIGN KEY (promoter) REFERENCES users(id); -- required by constraint rpminfo_build_id_draft_fkey on table rpminfo ALTER TABLE build ADD CONSTRAINT draft_for_rpminfo UNIQUE (id, draft); + ALTER TABLE build ADD CONSTRAINT promotion_sane CHECK (NOT draft OR (promotion_time IS NULL AND promoter IS NULL)); ALTER TABLE build ADD CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*,draft_' || id::TEXT || '$')) OR NOT draft); diff --git a/schemas/schema.sql b/schemas/schema.sql index f024122..2a61de8 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -290,9 +290,11 @@ CREATE TABLE build ( create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), start_time TIMESTAMPTZ, completion_time TIMESTAMPTZ, + promotion_time TIMESTAMPTZ, state INTEGER NOT NULL, task_id INTEGER REFERENCES task (id), owner INTEGER NOT NULL REFERENCES users (id), + promoter INTEGER REFERENCES users (id), cg_id INTEGER REFERENCES content_generator(id), extra TEXT, CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), @@ -300,6 +302,7 @@ CREATE TABLE build ( CONSTRAINT draft_for_rpminfo UNIQUE (id, draft), CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR (state <> 0 AND completion_time IS NOT NULL)), + CONSTRAINT promotion_sane CHECK (NOT draft OR (promotion_time IS NULL AND promoter IS NULL)), CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*,draft_' || id::TEXT || '$')) OR NOT draft) ) WITHOUT OIDS; From ce32f58b9a05d8614629b1f0fe74316f1e810835 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:01 +0000 Subject: [PATCH 65/70] show promotion info in ui --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 58026dc..3cb4772 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -3687,6 +3687,9 @@ def anon_handle_buildinfo(goptions, session, args): else: print("Task: none") print("Finished: %s" % koji.formatTimeLong(info['completion_ts'])) + if info.get('promotion_ts'): + print("Promoted by: %(promoter_name)s" % info) + print("Promoted at: %s" % koji.formatTimeLong(info['promotion_ts'])) maven_info = session.getMavenBuild(info['id']) if maven_info: print("Maven groupId: %s" % maven_info['group_id']) diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 5b1fbe0..e7a45e8 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -26,7 +26,7 @@ #if $build.draft DraftTrue - #else + #else DraftFalse #end if #if $build.get('source') @@ -97,6 +97,14 @@ Completed$util.formatTimeLong($build.completion_ts) #end if + #if $build.promotion_ts + + Promoted$util.formatTimeLong($build.promotion_ts) + + + Promoted by$util.escapeHTML($build.promoter_name) + + #end if #if $build.cg_id Content generator$util.escapeHTML($build.cg_name) From c5395a5a8f90340e551e097b147828e614d217d5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:02 +0000 Subject: [PATCH 66/70] fix unit tests - adjustments for query updates - draft data no longer in build.extra - drop some unhelpful assertions --- diff --git a/tests/test_hub/test_get_build.py b/tests/test_hub/test_get_build.py index ac4fe1e..32f0ecf 100644 --- a/tests/test_hub/test_get_build.py +++ b/tests/test_hub/test_get_build.py @@ -31,21 +31,7 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['build']) - self.assertEqual(query.joins, ['events ON build.create_event = events.id', - 'package on build.pkg_id = package.id', - 'volume on build.volume_id = volume.id', - 'users on build.owner = users.id']) self.assertEqual(query.clauses, ['build.id = %(buildID)i']) - self.assertEqual(query.columns, - ['build.id', 'build.cg_id', 'build.completion_time', - "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.draft', 'build.epoch', - 'build.extra', 'build.id', 'package.name', - "package.name || '-' || build.version || '-' || build.release", - 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', - 'build.source', 'build.start_time', - "date_part('epoch', build.start_time)", 'build.state', 'build.task_id', - 'build.version', 'volume.id', 'volume.name']) def test_non_exist_build_int_without_result_without_strict(self): build = 11 @@ -56,21 +42,7 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['build']) - self.assertEqual(query.joins, ['events ON build.create_event = events.id', - 'package on build.pkg_id = package.id', - 'volume on build.volume_id = volume.id', - 'users on build.owner = users.id']) self.assertEqual(query.clauses, ['build.id = %(buildID)i']) - self.assertEqual(query.columns, - ['build.id', 'build.cg_id', 'build.completion_time', - "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.draft', 'build.epoch', - 'build.extra', 'build.id', 'package.name', - "package.name || '-' || build.version || '-' || build.release", - 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', - 'build.source', 'build.start_time', - "date_part('epoch', build.start_time)", 'build.state', 'build.task_id', - 'build.version', 'volume.id', 'volume.name']) def test_non_exist_build_dict_with_strict(self): build = { @@ -103,21 +75,7 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['build']) - self.assertEqual(query.joins, ['events ON build.create_event = events.id', - 'package on build.pkg_id = package.id', - 'volume on build.volume_id = volume.id', - 'users on build.owner = users.id']) self.assertEqual(query.clauses, ['build.id = %(buildID)i']) - self.assertEqual(query.columns, - ['build.id', 'build.cg_id', 'build.completion_time', - "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.draft', 'build.epoch', - 'build.extra', 'build.id', 'package.name', - "package.name || '-' || build.version || '-' || build.release", - 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', - 'build.source', 'build.start_time', - "date_part('epoch', build.start_time)", 'build.state', 'build.task_id', - 'build.version', 'volume.id', 'volume.name']) def test_result_with_cg_id(self): build = 11 @@ -130,18 +88,4 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] self.assertEqual(query.tables, ['build']) - self.assertEqual(query.joins, ['events ON build.create_event = events.id', - 'package on build.pkg_id = package.id', - 'volume on build.volume_id = volume.id', - 'users on build.owner = users.id']) self.assertEqual(query.clauses, ['build.id = %(buildID)i']) - self.assertEqual(query.columns, - ['build.id', 'build.cg_id', 'build.completion_time', - "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.draft', 'build.epoch', - 'build.extra', 'build.id', 'package.name', - "package.name || '-' || build.version || '-' || build.release", - 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', - 'build.source', 'build.start_time', - "date_part('epoch', build.start_time)", 'build.state', 'build.task_id', - 'build.version', 'volume.id', 'volume.name']) diff --git a/tests/test_hub/test_list_builds.py b/tests/test_hub/test_list_builds.py index 356ca2d..239a1dc 100644 --- a/tests/test_hub/test_list_builds.py +++ b/tests/test_hub/test_list_builds.py @@ -17,32 +17,48 @@ class TestListBuilds(unittest.TestCase): return query def setUp(self): - # defaults self.tables= ['build'] + # note: QueryProcessor reports these sorted by alias self.columns = [ - 'build.id', 'build.completion_time', + 'build.id', + 'build.completion_time', "date_part('epoch', build.completion_time)", - 'events.id', 'events.time', + 'events.id', + 'events.time', "date_part('epoch', events.time)", 'build.draft', 'build.epoch', - 'build.extra', 'package.name', - "package.name || '-' || build.version || '-' || " - "build.release", 'users.id', 'users.name', 'package.id', - 'package.name', 'build.release', 'build.source', - 'build.start_time', "date_part('epoch', build.start_time)", - 'build.state', 'build.task_id', 'build.version', - 'volume.id', 'volume.name' + 'build.extra', + 'package.name', + "package.name || '-' || build.version || '-' || build.release", + 'users.id', + 'users.name', + 'package.id', + 'package.name', + 'promoter.id', + 'promoter.name', + 'build.promotion_time', + "date_part('epoch', build.promotion_time)", + 'build.release', + 'build.source', + 'build.start_time', + "date_part('epoch', build.start_time)", + 'build.state', + 'build.task_id', + 'build.version', + 'volume.id', + 'volume.name', ] self.clauses = ['package.id = %(packageID)i'] self.joins = [ 'LEFT JOIN events ON build.create_event = events.id', 'LEFT JOIN package ON build.pkg_id = package.id', 'LEFT JOIN volume ON build.volume_id = volume.id', - 'LEFT JOIN users ON build.owner = users.id' + 'LEFT JOIN users ON build.owner = users.id', + 'LEFT JOIN users AS promoter ON build.promoter = promoter.id', ] - + self.maxDiff = None self.exports = kojihub.RootExports() self.query_executeOne = mock.MagicMock() diff --git a/tests/test_hub/test_new_build.py b/tests/test_hub/test_new_build.py index 758e0da..5b8f6f3 100644 --- a/tests/test_hub/test_new_build.py +++ b/tests/test_hub/test_new_build.py @@ -179,8 +179,8 @@ class TestNewBuild(unittest.TestCase): insert_data = { "completion_time": "NOW", "epoch": "test_epoch", - "extra": '{"draft": {"target_release": "test_release", "promoted": false}}', "id": 108, + "extra": None, "owner": 123, "pkg_id": 54, "release": "test_release,draft_108", diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py index b554e66..e18f0a3 100644 --- a/tests/test_hub/test_promote_build.py +++ b/tests/test_hub/test_promote_build.py @@ -39,9 +39,6 @@ class TestPromoteBuild(unittest.TestCase): self.list_tags = mock.patch('kojihub.kojihub.list_tags', return_value=[{'id': 101}]).start() self.set_tag_update = mock.patch('kojihub.kojihub.set_tag_update').start() - self.encode_datetime = mock.patch( - 'kojihub.kojihub.encode_datetime', return_value='NOW' - ).start() self._now = datetime.datetime.now() self._datetime = mock.patch('kojihub.kojihub.datetime.datetime').start() self.now = self._datetime.now = mock.MagicMock(return_value=self._now) @@ -52,11 +49,6 @@ class TestPromoteBuild(unittest.TestCase): 'version': 'bar', 'release': 'tgtrel,draft_1', 'nvr': 'testnvr', - 'extra': { - 'draft': { - 'promoted': False, - 'target_release': 'tgtrel' - }}, 'state': 1, 'draft': True, 'volume_id': 99, @@ -83,19 +75,6 @@ class TestPromoteBuild(unittest.TestCase): self.new_build ] - extra = json.dumps( - { - 'draft': { - 'promoted': True, - 'target_release': 'tgtrel', - 'old_release': 'tgtrel,draft_1', - 'promotion_time': 'NOW', - 'promotion_ts': self._now.timestamp(), - 'promoter': self.user['name'] - } - } - ) - ret = self.exports.promoteBuild('a-draft-build') self.assertEqual(ret, self.new_build) self.assertEqual(len(self.updates), 1) @@ -103,9 +82,9 @@ class TestPromoteBuild(unittest.TestCase): self.assertEqual(update.table, 'build') self.assertEqual(update.values, self.draft_build) self.assertEqual(update.data, {'draft': False, - 'release': 'tgtrel', - 'extra': extra}) - self.assertEqual(update.rawdata, {}) + 'promoter': self.user['id'], + 'release': 'tgtrel'}) + self.assertEqual(update.rawdata, {'promotion_time': 'now()'}) self.assertEqual(update.clauses, ['id=%(id)i']) self.apply_volume_policy.assert_called_once_with( self.new_build, strict=False @@ -138,12 +117,6 @@ class TestPromoteBuild(unittest.TestCase): # bad delimiter 'release': 'tgtrel@draft_1', 'nvr': 'testnvr', - 'extra': { - 'draft': { - 'promoted': False, - # target_release doesn't matter now - 'target_release': 'any' - }}, 'state': 1, 'draft': True, 'volume_id': 99, @@ -168,11 +141,6 @@ class TestPromoteBuild(unittest.TestCase): 'version': 'bar', 'release': 'tgtrel#draft_1', 'nvr': 'testnvr', - 'extra': { - 'draft': { - 'promoted': False, - 'target_release': 'any' - }}, 'draft': True, 'state': 0, 'volume_id': 99, diff --git a/tests/test_hub/test_read_tagged_builds.py b/tests/test_hub/test_read_tagged_builds.py index f919df0..e48df0b 100644 --- a/tests/test_hub/test_read_tagged_builds.py +++ b/tests/test_hub/test_read_tagged_builds.py @@ -6,6 +6,8 @@ import koji import kojihub import copy +from koji.util import dslice + QP = kojihub.QueryProcessor @@ -29,35 +31,32 @@ class TestReadTaggedBuilds(unittest.TestCase): self.readPackageList = mock.patch('kojihub.kojihub.readPackageList').start() self.lookup_name = mock.patch('kojihub.kojihub.lookup_name').start() self.tag_name = 'test-tag' - self.columns = ['tag.id', 'tag.name', 'build.id', 'build.version', 'build.release', - 'build.epoch', 'build.state', 'build.completion_time', 'build.start_time', - 'build.task_id', 'build.draft', 'users.id', 'users.name', 'events.id', 'events.time', - 'volume.id', 'volume.name', 'package.id', 'package.name', - 'package.name || \'-\' || build.version || \'-\' || build.release', - 'tag_listing.create_event'] - self.fields = [('tag.id', 'tag_id'), ('tag.name', 'tag_name'), ('build.id', 'id'), - ('build.id', 'build_id'), ('build.version', 'version'), - ('build.release', 'release'), ('build.epoch', 'epoch'), - ('build.state', 'state'), ('build.completion_time', 'completion_time'), - ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), - ('build.draft', 'draft'), - ('users.id', 'owner_id'), ('users.name', 'owner_name'), - ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), - ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), - ('package.id', 'package_id'), ('package.name', 'package_name'), - ('package.name', 'name'), - ("package.name || '-' || build.version || '-' || build.release", 'nvr'), - ('tag_listing.create_event', 'create_event')] + self.columns = ['build.id', 'build.completion_time', 'tag_listing.create_event', + 'events.id', 'events.time', 'build.draft', 'build.epoch', 'build.id', + 'package.name', + "package.name || '-' || build.version || '-' || build.release", + 'users.id', 'users.name', 'package.id', 'package.name', 'promoter.id', + 'promoter.name', 'build.promotion_time', 'build.release', + 'build.start_time', 'build.state', 'tag.id', 'tag.name', 'build.task_id', + 'build.version', 'volume.id', 'volume.name'] + self.values = {'owner': None, + 'package': None, + 'st_complete': 1, + 'tagid': self.tag_name, + } self.joins = ['tag ON tag.id = tag_listing.tag_id', 'build ON build.id = tag_listing.build_id', 'events ON events.id = build.create_event', 'package ON package.id = build.pkg_id', 'volume ON volume.id = build.volume_id', - 'users ON users.id = build.owner', ] - self.aliases = ['tag_id', 'tag_name', 'id', 'build_id', 'version', 'release', 'epoch', - 'state', 'completion_time', 'start_time', 'task_id', 'draft', 'owner_id', - 'owner_name', 'creation_event_id', 'creation_time', 'volume_id', - 'volume_name', 'package_id', 'package_name', 'name', 'nvr', 'create_event'] + 'users ON users.id = build.owner', + 'LEFT JOIN users AS promoter ON promoter.id = build.promoter', + ] + self.aliases = ['build_id', 'completion_time', 'create_event', 'creation_event_id', + 'creation_time', 'draft', 'epoch', 'id', 'name', 'nvr', 'owner_id', + 'owner_name', 'package_id', 'package_name', 'promoter_id', + 'promoter_name', 'promotion_time', 'release', 'start_time', 'state', + 'tag_id', 'tag_name', 'task_id', 'version', 'volume_id', 'volume_name'] self.clauses = ['(tag_listing.active = TRUE)', 'tag_id = %(tagid)s', 'build.state = %(st_complete)i'] @@ -79,19 +78,14 @@ class TestReadTaggedBuilds(unittest.TestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - values = {'clauses': self.clauses, 'event': None, 'extra': False, 'fields': self.fields, - 'inherit': False, 'joins': self.joins, 'latest': False, 'owner': None, - 'package': None, 'packages': self.package_list, - 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, - 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': None, 'draft': None - } + values = self.values.copy() self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, self.joins) self.assertEqual(set(query.columns), set(self.columns)) self.assertEqual(set(query.aliases), set(self.aliases)) self.assertEqual(set(query.clauses), set(self.clauses)) - self.assertEqual(query.values, values) + # function passes values=locals(), so we only check the relevant values + self.assertEqual(dslice(query.values, values.keys()), values) def test_get_tagged_builds_package_owner_type_maven_extra(self): self.readPackageList.return_value = self.package_list @@ -106,27 +100,21 @@ class TestReadTaggedBuilds(unittest.TestCase): 'maven_builds.version', 'build.extra']) aliases = copy.deepcopy(self.aliases) aliases.extend(['maven_group_id', 'maven_artifact_id', 'maven_version', 'extra']) - fields = copy.deepcopy(self.fields) - fields.extend([('maven_builds.group_id', 'maven_group_id'), - ('maven_builds.artifact_id', 'maven_artifact_id'), - ('maven_builds.version', 'maven_version'), ('build.extra', 'extra')]) clauses = copy.deepcopy(self.clauses) clauses.extend(['package.name = %(package)s', 'users.name = %(owner)s']) joins = copy.deepcopy(self.joins) joins.append('maven_builds ON maven_builds.build_id = tag_listing.build_id') - values = {'clauses': clauses, 'event': None, 'extra': True, 'fields': fields, - 'inherit': False, 'joins': joins, 'latest': False, 'owner': self.username, - 'package': self.pkg_name, 'packages': self.package_list, - 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, - 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'maven', 'draft': None} + values = self.values.copy() + values['owner'] = self.username + values['package'] = self.pkg_name self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) self.assertEqual(set(query.aliases), set(aliases)) self.assertEqual(set(query.clauses), set(clauses)) - self.assertEqual(query.values, values) + # function passes values=locals(), so we only check the relevant values + self.assertEqual(dslice(query.values, values.keys()), values) def test_get_tagged_builds_type_win_latest(self): self.readPackageList.return_value = self.package_list @@ -139,23 +127,17 @@ class TestReadTaggedBuilds(unittest.TestCase): columns.append('win_builds.platform') aliases = copy.deepcopy(self.aliases) aliases.append('platform') - fields = copy.deepcopy(self.fields) - fields.append(('win_builds.platform', 'platform')) joins = copy.deepcopy(self.joins) joins.append('win_builds ON win_builds.build_id = tag_listing.build_id') - values = {'clauses': self.clauses, 'event': None, 'extra': False, 'fields': fields, - 'inherit': False, 'joins': joins, 'latest': True, 'owner': None, - 'package': None, 'packages': self.package_list, - 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, - 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'win', 'draft': None} + values = self.values.copy() self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) self.assertEqual(set(query.aliases), set(aliases)) self.assertEqual(set(query.clauses), set(self.clauses)) - self.assertEqual(query.values, values) + # function passes values=locals(), so we only check the relevant values + self.assertEqual(dslice(query.values, values.keys()), values) def test_get_tagged_builds_type_image(self): self.readPackageList.return_value = self.package_list @@ -168,23 +150,17 @@ class TestReadTaggedBuilds(unittest.TestCase): columns.append('image_builds.build_id') aliases = copy.deepcopy(self.aliases) aliases.append('build_id') - fields = copy.deepcopy(self.fields) - fields.append(('image_builds.build_id', 'build_id')) joins = copy.deepcopy(self.joins) joins.append('image_builds ON image_builds.build_id = tag_listing.build_id') - values = {'clauses': self.clauses, 'event': None, 'extra': False, 'fields': fields, - 'inherit': False, 'joins': joins, 'latest': False, 'owner': None, - 'package': None, 'packages': self.package_list, - 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, - 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'image', 'draft': None} + values = self.values.copy() self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) self.assertEqual(set(query.aliases), set(aliases)) self.assertEqual(set(query.clauses), set(self.clauses)) - self.assertEqual(query.values, values) + # function passes values=locals(), so we only check the relevant values + self.assertEqual(dslice(query.values, values.keys()), values) def test_get_tagged_builds_type_non_exist(self): self.readPackageList.return_value = self.package_list @@ -207,19 +183,14 @@ class TestReadTaggedBuilds(unittest.TestCase): joins = copy.deepcopy(self.joins) joins.append('build_types ON build.id = build_types.build_id AND btype_id = %(btype_id)s') - values = {'btype': typeinfo, 'btype_id': typeinfo['id'], 'clauses': self.clauses, - 'event': None, 'extra': False, 'fields': self.fields, - 'inherit': False, 'joins': joins, 'latest': False, 'owner': None, - 'package': None, 'packages': self.package_list, - 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, - 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': type, 'draft': None} + values = self.values.copy() self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(self.columns)) self.assertEqual(set(query.aliases), set(self.aliases)) self.assertEqual(set(query.clauses), set(self.clauses)) - self.assertEqual(query.values, values) + # function passes values=locals(), so we only check the relevant values + self.assertEqual(dslice(query.values, values.keys()), values) def test_get_tagged_builds_draft(self): self.readPackageList.return_value = self.package_list @@ -231,17 +202,11 @@ class TestReadTaggedBuilds(unittest.TestCase): clauses = copy.deepcopy(self.clauses) clauses.extend(['draft IS TRUE']) - values = {'clauses': clauses, 'event': None, 'extra': False, 'fields': self.fields, - 'inherit': False, 'joins': self.joins, 'latest': False, 'owner': None, - 'package': None, 'packages': self.package_list, - 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, - 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': None, 'draft': True - } - + values = self.values.copy() self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, self.joins) self.assertEqual(set(query.columns), set(self.columns)) self.assertEqual(set(query.aliases), set(self.aliases)) self.assertEqual(set(query.clauses), set(clauses)) - self.assertEqual(query.values, values) + # function passes values=locals(), so we only check the relevant values + self.assertEqual(dslice(query.values, values.keys()), values) From c944ef36a1b3ed1c387cbd0af6b29c2aae953ff5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:02 +0000 Subject: [PATCH 67/70] unit test for get_rpm preferences --- diff --git a/tests/test_hub/test_getRPM.py b/tests/test_hub/test_getRPM.py index ef15d46..f9ab508 100644 --- a/tests/test_hub/test_getRPM.py +++ b/tests/test_hub/test_getRPM.py @@ -121,6 +121,45 @@ class TestGetRPM(DBQueryTestCase): ['external_repo ON rpminfo.external_repo_id = external_repo.id']) self.assertEqual(query.values, rpminfo_data) + @mock.patch('kojihub.kojihub._get_rpms') + def test_rpm_info_preferred(self, _get_rpms): + rpminfo = 'testrpm-1.23-4.x86_64.rpm' + rpm_ext = {'id': 1, 'external_repo_id': 2, 'draft': False} + rpm_ext2 = {'id': 2, 'external_repo_id': 2, 'draft': False} + rpm_draft = {'id': 3, 'external_repo_id': 0, 'draft': True} + rpm_draft2 = {'id': 4, 'external_repo_id': 0, 'draft': True} + rpm_nondraft = {'id': 5, 'external_repo_id': 0, 'draft': False} + + # this test checks that the expected preferences are followed for multiple matches + + _get_rpms.return_value = [rpm_ext, rpm_ext2, rpm_draft, rpm_draft2, rpm_nondraft] + ret = kojihub.get_rpm(rpminfo, multi=False) + self.assertEqual(ret, rpm_nondraft) + + _get_rpms.return_value = [rpm_ext, rpm_ext2, rpm_draft, rpm_draft2] + ret = kojihub.get_rpm(rpminfo, multi=False) + self.assertEqual(ret, rpm_draft2) + + _get_rpms.return_value = [rpm_ext, rpm_ext2, rpm_draft] + ret = kojihub.get_rpm(rpminfo, multi=False) + self.assertEqual(ret, rpm_draft) + + _get_rpms.return_value = [rpm_ext, rpm_ext2] + ret = kojihub.get_rpm(rpminfo, multi=False) + self.assertEqual(ret, rpm_ext2) + + _get_rpms.return_value = [rpm_ext] + ret = kojihub.get_rpm(rpminfo, multi=False) + self.assertEqual(ret, rpm_ext) + + # multiple nondraft matches should error + rpm_bad = {'id': 6, 'external_repo_id': 0, 'draft': False} + _get_rpms.return_value = [rpm_nondraft, rpm_bad] + with self.assertRaises(koji.GenericError): + ret = kojihub.get_rpm(rpminfo, multi=False) + + self.assertEqual(len(self.queries), 0) # _get_rpm is mocked + class TestGetRPMHeaders(unittest.TestCase): From 387dbecebe96689a4f8b9730179765d5cd527870 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:02 +0000 Subject: [PATCH 68/70] simplify draft_release_sane logic --- diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index 8437b5e..a4315ae 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -61,8 +61,7 @@ BEGIN; ALTER TABLE build ADD CONSTRAINT draft_for_rpminfo UNIQUE (id, draft); ALTER TABLE build ADD CONSTRAINT promotion_sane CHECK (NOT draft OR (promotion_time IS NULL AND promoter IS NULL)); ALTER TABLE build ADD CONSTRAINT draft_release_sane CHECK - ((draft AND release ~ ('^.*,draft_' || id::TEXT || '$')) - OR NOT draft); + (NOT draft OR release ~ ('^.*,draft_' || id::TEXT || '$')); ALTER TABLE rpminfo ADD COLUMN draft BOOLEAN; UPDATE rpminfo SET draft=FALSE WHERE build_id IS NOT NULL; diff --git a/schemas/schema.sql b/schemas/schema.sql index 2a61de8..d74535d 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -303,8 +303,7 @@ CREATE TABLE build ( CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR (state <> 0 AND completion_time IS NOT NULL)), CONSTRAINT promotion_sane CHECK (NOT draft OR (promotion_time IS NULL AND promoter IS NULL)), - CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*,draft_' || id::TEXT || '$')) OR - NOT draft) + CONSTRAINT draft_release_sane CHECK (NOT draft OR release ~ ('^.*,draft_' || id::TEXT || '$')) ) WITHOUT OIDS; CREATE INDEX build_by_pkg_id ON build (pkg_id); From cf605842d8ecb0bc76b5b52687f7977965e18f4b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jan 05 2024 10:01:03 +0000 Subject: [PATCH 69/70] schema comments --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 036e458..0cdb418 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -10311,6 +10311,8 @@ def _promote_build(build, force=False): update.set(draft=False, release=target_release, promoter=user['id']) update.rawset(promotion_time='now()') update.execute() + # Note: changing build.draft will implicitly update corresponding rpminfo.draft fields + # due to the ON UPDATE CASCADE foreign key constraint new_binfo = get_build(binfo['id'], strict=True) oldpath = koji.pathinfo.build(binfo) diff --git a/schemas/schema.sql b/schemas/schema.sql index d74535d..e5f3462 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -298,8 +298,8 @@ CREATE TABLE build ( cg_id INTEGER REFERENCES content_generator(id), extra TEXT, CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), - -- required by constraint rpminfo_build_id_draft_fkey on table rpminfo CONSTRAINT draft_for_rpminfo UNIQUE (id, draft), +-- ^ required by constraint rpminfo_build_id_draft_fkey on table rpminfo CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR (state <> 0 AND completion_time IS NOT NULL)), CONSTRAINT promotion_sane CHECK (NOT draft OR (promotion_time IS NULL AND promoter IS NULL)), @@ -725,7 +725,7 @@ CREATE TABLE group_package_listing ( -- rpminfo tracks individual rpms (incl srpms) -- buildroot_id can be NULL (for externally built packages) --- even though we track epoch, we demand that N-V-R.A be unique +-- even though we track epoch, we demand that N-V-R.A be unique (for non-draft builds) -- we don't store filename b/c filename should be N-V-R.A.rpm CREATE TABLE rpminfo ( id SERIAL NOT NULL PRIMARY KEY, @@ -744,6 +744,7 @@ CREATE TABLE rpminfo ( metadata_only BOOLEAN NOT NULL DEFAULT FALSE, extra TEXT, FOREIGN KEY (build_id, draft) REFERENCES build (id, draft) ON UPDATE CASCADE, +-- ^ ensures the draft field is consistent with the build entry CONSTRAINT build_id_draft_external_repo_id_sane CHECK ( (draft IS NULL AND build_id IS NULL AND external_repo_id <> 0) OR (draft IS NOT NULL AND build_id IS NOT NULL AND external_repo_id = 0)) From e71c8d48cd0903135517547f1c4c5a610f9893f6 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jan 07 2024 00:10:13 +0000 Subject: [PATCH 70/70] fix flake8 --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 0cdb418..4ea9163 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -6099,10 +6099,8 @@ def new_build(data, strict=False): for f in ('version', 'release', 'epoch'): if f not in data: raise koji.GenericError("No %s value for build" % f) - extra = {} if 'extra' in data: try: - extra = data['extra'] # backwards compatible for data in callback data['extra'] = json.dumps(data['extra']) except Exception: @@ -6145,7 +6143,6 @@ def new_build(data, strict=False): data['id'] = insert_data['id'] = nextval('build_id_seq') # handle draft suffix in release if data.get('draft'): - target_release = data['release'] data['release'] = insert_data['release'] = koji.gen_draft_release( data['release'], data['id'] )