From b9c1af4b7a3d143064f927d27c78829a9a1b30fb Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 1/14] add new indexes in schema upgrade --- diff --git a/docs/schema-upgrade-1.17-1.18.sql b/docs/schema-upgrade-1.17-1.18.sql index 16ae84a..751d7d7 100644 --- a/docs/schema-upgrade-1.17-1.18.sql +++ b/docs/schema-upgrade-1.17-1.18.sql @@ -34,4 +34,12 @@ CREATE INDEX build_reservations_created ON build_reservations(created); ALTER TABLE build ADD COLUMN cg_id INTEGER REFERENCES content_generator(id); + +-- new indexes added in 1.18 +CREATE INDEX tag_packages_active_tag_id ON tag_packages(active, tag_id); +CREATE INDEX tag_packages_create_event ON tag_packages(create_event); +CREATE INDEX tag_packages_revoke_event ON tag_packages(revoke_event); +CREATE INDEX tag_packages_owner ON tag_packages(owner); + + COMMIT; From 949e17f279e7e5049f4e47c387c8879b004e689d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 2/14] migration doc --- diff --git a/docs/source/migrating_to_1.18.rst b/docs/source/migrating_to_1.18.rst new file mode 100644 index 0000000..353046c --- /dev/null +++ b/docs/source/migrating_to_1.18.rst @@ -0,0 +1,33 @@ +Migrating to Koji 1.18 +====================== + +.. + reStructured Text formatted + +You should consider the following changes when migrating to 1.18: + +DB Updates +---------- + +This release has a few schema changes: + + * Several new indexes to speed operations + * A ``cg_id`` field has been added to the ``build`` table + * A new ``build_reservations`` table + * A new ``build_notifications_block`` table + * Updates to the data in the ``archivetypes`` table + +As in previous releases, we provide a migration script that updates the +database. + +:: + + # psql koji koji Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 3/14] bump version, add changelog --- diff --git a/docs/source/conf.py b/docs/source/conf.py index 871ec45..09d63f5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = u'2017, Mike McLean, Mike B, Dennis Gilmore, Mathieu Bridon, Ian McL # built documents. # # The short X.Y version. -version = '1.17' +version = '1.18' # The full version, including alpha/beta/rc tags. -release = '1.17.0' +release = '1.18.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/koji.spec b/koji.spec index 5a6a757..75b3548 100644 --- a/koji.spec +++ b/koji.spec @@ -81,7 +81,7 @@ %define release %{baserelease} %endif Name: koji -Version: 1.17.0 +Version: 1.18.0 Release: %{release}%{?dist} License: LGPLv2 and GPLv2+ # the included arch lib from yum's rpmUtils is GPLv2+ @@ -713,6 +713,108 @@ fi %endif %changelog +* Fri Aug 2 2019 Mike McLean - 1.18.0-1 +- PR#1464: API for reserving NVRs for content generators +- PR#898: Add support for tag/target macros for Mageia +- PR#1544: use RawConfigParser for kojid +- PR#863: cli: change --force to real bool arg for add-tag-inheritance +- PR#1253: cli: add option for custom cert location +- PR#1353: Create db index for listTagged +- PR#1375: docs: add architecture diagram +- PR#892: cli: also load plugins from ~/.koji/plugins +- PR#1516: kojibuilder: Pass mergerepo_c --all for bare mode as well. +- PR#1524: set module_hotfixes=1 in yum.conf via tag config +- PR#1417: notification's optouts +- PR#1515: add debug message to new multicall to match original +- PR#1480: Add raw-gz and compressed QCOW2 archive types. +- PR#1260: use LANG=C for running all tests +- PR#1447: handle deleted tags in kojira +- PR#1513: Allow hub policy to match version and release +- PR#1462: rebuildSRPM task +- PR#1498: Pass bytes to md5_constructor +- PR#1502: Don't pass block list in bare merge mode +- PR#1489: pass bytes to sha1 constructor +- PR#1499: remove merge option from edit-external-repo +- PR#1427: Fix typo in getArchiveTypes docstring +- PR#957: New multicall interface +- PR#1280: put fix_pyver before printing command help +- PR#1415: New 'buildtype' test for policies +- PR#1258: retain old search pattern in web ui +- PR#1479: use better index for sessions +- PR#1279: let hub decide, what headers are supported +- PR#1454: introduce host-admin permission + docs +- PR#1303: fix history display for parallel host_channels updates +- PR#1278: createrepo_c is used by default now +- PR#1449: show load/capacity in list-channels +- PR#1476: Allow taginfo cli to use tag IDs; fixed Inheritance printing bug +- PR#1445: turn back on test skipped due to coverage bug +- PR#1452: fix parentheses for tuple in _writeInheritanceData +- PR#1456: deprecate BuildRoot.uploadDir method +- PR#1461: check existence of tag_id in getInheritanceData +- PR#1471: list-hosts shouldn't error on empty list +- PR#1273: Allow generating separate src repo for build repos +- PR#1255: always check existence of tag in setInheritanceData +- PR#1256: add strict option to getTaskChildren +- PR#1257: fail runroot task on non-existing tag +- PR#1272: check architecture names for mistakes +- PR#1322: Reduce duplicate "fixEncoding" code +- PR#1327: volume option for dist-repo +- PR#1442: delete_build: handle results of lazy build_references call +- PR#1425: add --show-channels listing to list-hosts +- PR#1432: py2.6 compatibility fix +- PR#1434: hub: fix check_fields and duplicated parent_id in _writeInheritanceData +- PR#1439: user correct column in sql (getTask) +- PR#1437: fix table name in build_references query +- PR#1414: Fix jenkins config for new python mock +- PR#1411: handle bare merge mode +- PR#1410: build_srpm: Wait until after running the sources command to check for alt_sources_dir +- PR#1383: display task durations in webui +- PR#1358: rollback errors in multiCall +- PR#1413: Makefile: print correct urls for test coverage +- PR#1409: Fix SQL after introduction of host_config +- PR#1324: createEmptyBuild errors for non-existent user +- PR#1406: fix mapping iteration in getFullInheritance +- PR#1398: kojid: Download only 'origin' +- PR#1365: Check CLI arguments for enable/disable host +- PR#1390: CLI list-channels sorted output +- PR#1389: block_pkglist compatibility fix +- PR#1376: use context manager for open in CLI +- PR#1392: Replace references to latest-pkg with latest-build +- PR#1386: scale task_avail_delay based on bin rank +- PR#1363: Use createrepo_update even for first repo run +- PR#1368: update test requirements in jenkins +- PR#1374: honor mock.package_manager tag setting in mock-config cli +- PR#1387: remove unused variable +- PR#1143: hub: document CG access method arguments +- PR#1169: docs: use systemctl enable --now for postgres and kojid +- PR#1155: hub: document addHost and editHost arguments +- PR#1242: kojid.conf documentation +- PR#1340: Update server doc for newer TLS and event worker +- PR#1359: docs: remove "TBD" sections +- PR#1360: docs: remove mod_python references +- PR#1361: docs: kojirepod -> kojira +- PR#1370: add vhdx archivetype +- PR#1331: provide lower level versions of build_target functions +- PR#1348: rm old references to Mozilla +- PR#1297: Support tilde in search +- PR#1356: kojira: fix iteration over repos in py3 +- PR#1342: Remove python2.4 OptionParse fix +- PR#1347: Fix hub startup handling +- PR#1346: Rely on ozif_enabled switch in BaseImageTask +- PR#1344: add .tgz to list of tar's possible extensions +- PR#1086: hub: unittest for get_external_repos +- PR#1170: docs: koji package provides schema.sql file +- PR#1281: remove urlescape from package name +- PR#1304: hub: document setInheritanceData arguments +- PR#1277: Remove 'keepalive' option +- PR#1330: fix docs typos +- PR#1339: fix typo in usage of six's import of MIMEText +- PR#1337: minor gc optimizations +- PR#1254: doc: Include AnyStor mention to 'koji runs here' doc +- PR#1325: run py3 tests in CI by default +- PR#1326: README: link to Pungi project instead of mash +- PR#1329: Update plugin doc (confusing sentence) + * Wed Mar 6 2019 Mike McLean - 1.17.0-1 - PR#1320: also remove nonprintable changelog chars in py3 - PR#1293: fix dict encoding in py3 From 01199c00ddf76cf0933d02c575edc3a09ce1ff44 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 4/14] initial release notes --- diff --git a/docs/source/migrations.rst b/docs/source/migrations.rst index 397b21c..f594d52 100644 --- a/docs/source/migrations.rst +++ b/docs/source/migrations.rst @@ -5,6 +5,7 @@ Migrations .. toctree:: :maxdepth: 1 + migrating_to_1.18 migrating_to_1.17 migrating_to_1.16 migrating_to_1.15 diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 2afd02e..e1ea496 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -5,6 +5,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release_notes_1.18 release_notes_1.17 release_notes_1.16.2 release_notes_1.16.1 diff --git a/docs/source/release_notes_1.18.rst b/docs/source/release_notes_1.18.rst new file mode 100644 index 0000000..1da66fc --- /dev/null +++ b/docs/source/release_notes_1.18.rst @@ -0,0 +1,1086 @@ +Koji 1.18.0 Release notes +========================= + + +Migrating from Koji 1.17 +------------------------ + +For details on migrating see :doc:`migrating_to_1.18` + + + +Security Fixes +-------------- + + + +Client Changes +-------------- + +**cli: add option for custom cert location** + +| PR: https://pagure.io/koji/pull-request/1253 + +The CLI no has an option for setting a custom SSL certificate, similar to the +options for Kerberos authentication. + + +**cli: load plugins from ~/.koji/plugins** + +| PR: https://pagure.io/koji/pull-request/892 + + +This change allows users to load their own cli plugins from ``~/.koji/plugins`` +or from another location by using the ``plugin_paths`` setting. + + + +Library Changes +--------------- + + +Web UI Changes +-------------- + + + +Builder Changes +--------------- + +**use RawConfigParser for kojid** + +| PR: https://pagure.io/koji/pull-request/1544 + +The use of percent signs is common in ``kojid.conf`` because of the +``host_principal_format`` setting. +This causes an error in python3 if ``SafeConfigParser`` is used, so we use +``RawConfigParser`` instead. + + +**handle bare merge mode** + +| PR: https://pagure.io/koji/pull-request/1411 +| PR: https://pagure.io/koji/pull-request/1516 +| PR: https://pagure.io/koji/pull-request/1502 + + +This feature adds a new merge mode for external repos named ``bare``. +This mode is intended for use with modularity. + + + + +System Changes +-------------- + + + +**API for reserving NVRs for content generators** + +| PR: https://pagure.io/koji/pull-request/1464 + +Fixes: https://pagure.io/koji/issue/1463 + +.. + https://pagure.io/koji/issue/1463 + [RFE] Predeclare nvr for content generators + +This feature allows content generators to reserve NVRs earlier in the build +process similar to builds performed by ``kojid``. The NVR is reserved by +calling ``CGInitBuild()`` and finalized by the ``CGImport()`` call. + + + +**Add support for tag/target macros for Mageia** + +| PR: https://pagure.io/koji/pull-request/898 + +This feature allows setting rpm macros via the tag extra field. These macros +will be added to the mock configuration for the buildroot. The system +looks for extra values of the form ``rpm.macro.NAME``. + +For example, to set the dist tag for a given tag, you could use a command like: + +:: + + $ koji edit-tag f30-build -x rpm.macro.dist=MYDISTTAG + + + +**set module_hotfixes=1 in yum.conf via tag config** + +| PR: https://pagure.io/koji/pull-request/1524 + +Koji now handles the field ``mock.yum.module_hotfixes`` in the tag extra. +When set, kojid will set ``module_hotfixes=0/1`` in the yum portion of the +mock configuration for a buildroot. + + +**Allow users to opt out of notifications** + +| PR: https://pagure.io/koji/pull-request/1417 + +This feature lets users opt out of notifications that they would otherwise +automatically recieve, such as build and tag notifications for: + +- the build owner (the user who submitted the build) +- the package owner within the given tag + +These opt-outs are user controlled and can be managed with the new +``block-notification`` and ``unblock-notificiation`` commands. + + + +**Allow hub policy to match version and release** + +| PR: https://pagure.io/koji/pull-request/1513 + + +This feature adds new policy tests to match ``version`` and ``release``. +This tests are glob pattern matches. + + +**Rebuild SPMS before building** + +| PR: https://pagure.io/koji/pull-request/1462 + +For rpm builds from an uploaded srpm, Koji will now rebuild the srpm in the +build environment first. +This ensures that the NVR is correct for the resulting build. + +The old behavior can be requested by setting ``rebuild_srpm=False`` in the tag +extra data for the build tag in question. + + + + +**remove merge option from edit-external-repo** + +| PR: https://pagure.io/koji/pull-request/1499 + +This option was mistakenly added to the command and never did anything. +It is gone now. + + +**New multicall interface** + +| PR: https://pagure.io/koji/pull-request/957 + +This feature implements a new and much better way to use multicall in the Koji +library. +These changes create a new implementation outside of ClientSession. +The old way will still work. + +With this new implementation: + +* a multicall is tracked as an instance of `MultiCallSession` +* the original session is unaffected +* multiple multicalls can be managed in parallel, if desired +* `MultiCallSession` behaves more or less like a session in multicall mode +* method calls return a `VirtualCall` instance that can later be used to access the result +* `MultiCallSession` can be used as a context manager, ensuring that the calls are executed + +Usage examples are availble in the :doc:`Writing Koji Code ` +document. + + + + +**New 'buildtype' test for policies** + +| PR: https://pagure.io/koji/pull-request/1415 + + +Koji added btypes in version 1.11 along with content generators. +Now, all builds have one or more btypes. + +This change allows policies to check the btype value using the ``buildtype`` test. + + + +**retain old search pattern in web ui** + +| PR: https://pagure.io/koji/pull-request/1258 + +The search results page of the web ui now retains a search form with the +current search pre-filled. +This makes it easier for users to refine their searches. + + + + + +**introduce host-admin permission + docs** + +| PR: https://pagure.io/koji/pull-request/1454 + + +I'm filing this as first of series to splitting admin permission to more granular ones. Adding some docs about permission system and `host`, `tag` and `target` permissions. + + + +**createrepo_c is used by default now** + +| PR: https://pagure.io/koji/pull-request/1278 + +Fixes: https://pagure.io/koji/issue/716 + +.. + https://pagure.io/koji/issue/716 + /usr/libexec/kojid/mergerepos ignores xml:base in location + +If you add an external repo to a build tag, and the external repo uses the same xml:base trickery that Koji itself uses (that is, if the external repo you are pointing it is produced by another Koji instance for a tag which itself has an external repo configure), the xml:base attribute is lost by /usr/libexec/kojid/mergerepos and the resulting merged repodata ends up with incorrect URLs. + +To give a hypothetical example, imagine your Koji is producing a repo from a build tag at: + +http://thirdpartykoji.example.com/kojifiles/repos/beaker-harness-rhel-8-build/latest/x86_64/repodata/ + +It uses an external repo to pull packages from another Koji instance at: + +http://download.bigcorp.com/brewroot/repos/rhel-8.0-build/latest/x86_64/repodata/ + +But that repo itself is pulling packages from an external repo: + +https://kojipkgs.fedoraproject.org/repos/module-bootstrap-rawhide/latest/x86_64/repodata/ + +The download.bigcorp.com repodata will refer to the remote packages using xml:base, like this: + + + +Everything is fine so far. But now, when mergerepos runs on thirdpartykoji.example.com, I expect it to preserve the same so that the packages can be correctly downloaded no matter how many layers of external repos are involved. However mergerepos on thirdpartykoji.example.com instead produces this: + + + +which fails to download because the package is not there. The original value of xml:base has been lost. + +The end result is build failures with errors like this in mock_output.log as yum/dnf tries to download the packages from the wrong URL: + + http://download.bigcorp.com/brewroot/repos/rhel-8.0-build/latest/i386/audit-libs-2.7.7-5.fc27.i686.rpm: [Errno 14] HTTP Error 404 - Not Found + Trying other mirror. + + + +**show load/capacity in list-channels** + +| PR: https://pagure.io/koji/pull-request/1449 + +Fixes: https://pagure.io/koji/issue/1448 + +.. + https://pagure.io/koji/issue/1448 + [RFE] show channel load/capacity in list-channels + +list-channels can show overall number for load/capacity + + + +**Allow taginfo cli to use tag IDs; fixed Inheritance printing bug** + +| PR: https://pagure.io/koji/pull-request/1476 + +It would be useful to be able to use the koji cli's taginfo with tag IDs and not just the tag name since the python library allows for this. Also there is a bug where Inheritance always uses the value of the last tag queried for. + +Old behavior (ID bug): +``` +❯❯❯ koji taginfo 6438 +No such tag: 6438 +``` + +Old behavior (Inheritance bug): +``` +❯❯❯ koji taginfo rawhide f31 +Tag: rawhide [197] +Arches: aarch64 armv7hl i686 ppc64 ppc64le s390x x86_64 +Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build +LOCKED +Tag options: +Inheritance: + <<<<< inheritance missing/incorrect + +Tag: f31 [6438] +Arches: None +Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build +Required permission: 'autosign' +Tag options: + mock.new_chroot : 0 + mock.package_manager : 'dnf' +Inheritance: + +``` +New behavior: +``` +❯❯❯ koji taginfo rawhide 6438 +Tag: rawhide [197] +Arches: aarch64 armv7hl i686 ppc64 ppc64le s390x x86_64 +Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build +LOCKED +Tag options: +Inheritance: + 0 .... f31 [6438] + +Tag: f31 [6438] +Arches: None +Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build +Required permission: 'autosign' +Tag options: + mock.new_chroot : 0 + mock.package_manager : 'dnf' +Inheritance: +``` + +.. + https://pagure.io/koji/issue/1485 + taginfo shows wrong inheritance when multiple tags given + +When multiple tags are requested, the inheritance data for the final tag is shown for all tags. + +``` +$ koji taginfo f31 f28-build +Tag: f31 [6438] +Arches: None +Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build +Required permission: 'autosign' +Tag options: + mock.new_chroot : 0 + mock.package_manager : 'dnf' +Inheritance: + 0 .... f28-override [1922] + +Tag: f28-build [1928] +Arches: armv7hl i686 x86_64 aarch64 ppc64 ppc64le s390x +Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build +Tag options: +Inheritance: + 0 .... f28-override [1922] +``` + +Here we see `f28`'s inheritance for both tags (in reality, `f31` has no inheritance). + + + + + + +**deprecate BuildRoot.uploadDir method** + +| PR: https://pagure.io/koji/pull-request/1456 + +Fixes: https://pagure.io/koji/issue/839 + +.. + https://pagure.io/koji/issue/839 + deprecate BuildRoot.uploadDir() + +Noticed while working on a patch that this method is not used + + + +**check existence of tag_id in getInheritanceData** + +| PR: https://pagure.io/koji/pull-request/1461 + +Fixes: https://pagure.io/koji/issue/1460 + +.. + https://pagure.io/koji/issue/1460 + getInheritanceData API call should raise GenericError exception for non existing tag ID + +*Steps to Reproduce:* +Run a getInheritanceData(non_exist_tag_ID) + +*Actual result:* + +[] + +*Expected result:* + +GenericError exception should be raised + + + + + + +**Allow generating separate src repo for build repos** + +| PR: https://pagure.io/koji/pull-request/1273 + +Fixes #1266 + +Currently Koji has an option that can be used to include source rpms in each of generated arch repos. Howewer it makes repositories significantly bigger and slower to generate. Source metadata is duplicated across all arch repos. Increased metadata size makes builds slower. + +This pull request takes a slightly different approach - it makes it possible to generate build repos with source rpms in separate repos. Such repos don't change size of each arch repo and can be generated faster - generation is done in a separate createrepo task that can run on separate host. For example: + + newRepo + ├ createrepo (src) + ├ createrepo (aarch64) + ├ createrepo (ppc64le) + └ createrepo (x86_64) + +CC @ignatenkobrain + +.. + https://pagure.io/koji/issue/1266 + RFE: --with-separate-src for build-repos + +In Fedora we want to have src in build repos, but in separate repo so the size of metadata for builds won't grow. + + + + + +**add strict option to getTaskChildren** + +| PR: https://pagure.io/koji/pull-request/1256 + ++ moving to QueryProcessor of used calls + +.. + https://pagure.io/koji/issue/1199 + API Call getTaskChildren call should return GenericError for non existing taskID + +*Steps to Reproduce:* + +Run getTaskChildren(non_existing_id) + +*Actual result:* + +[] + +*Expected result:* + +Should be returning an error message that task id does not exist. + + +I suggest add a strict option for the getTaskChildren API call. + + + +**fail runroot task on non-existing tag** + +| PR: https://pagure.io/koji/pull-request/1257 + +Fixes: https://pagure.io/koji/issue/1139 + +.. + https://pagure.io/koji/issue/1139 + runroot API call should raise GenericError exception for non existing tag + +*Steps to Reproduce:* + +tag = 'non-existing-tag' +arch 'x86_64' +Run runroot(tag, arch, ['echo', 'hello', 'world'], ) + +*Actual result:* + +API call doesn't return some message +Web interface returns this after the click to runroot task: + +Error +An error has occurred while processing your request. +IndexError: list index out of range +Full tracebacks disabled + +*Expected result:* + +Should be returning an error message that tag is not existing and on web interface should be normal task info with the failed state. +[![Screenshot_from_2018-10-31_09-50-46.png](/koji/issue/raw/files/b1dde9720c1fac719c568be125886ae087ec391a9911ed2e30fc3c228d7c452d-Screenshot_from_2018-10-31_09-50-46.png)](/koji/issue/raw/files/b1dde9720c1fac719c568be125886ae087ec391a9911ed2e30fc3c228d7c452d-Screenshot_from_2018-10-31_09-50-46.png) + + + + +**check architecture names for mistakes** + +| PR: https://pagure.io/koji/pull-request/1272 + +Fixes: https://pagure.io/koji/issue/1237 + +.. + https://pagure.io/koji/issue/1237 + Architectures field in build target uses 2 different separators + +Koji returns inconsistent data for architectures from build target, because both space and comma separators are allowed. + +Could be possible to unify separators to space character only (at least in API responses)? + +Ideally it would be great to get same order of the architectures no matter in which order they have been provided (e.g. sort them alphabetically). + + + + +**volume option for dist-repo** + +| PR: https://pagure.io/koji/pull-request/1327 + +When the rpms live on a different volume, it's very inefficient to generate a dist repo on the main volume (because rpms will have to be copied instead of linked). This change allows the user to specify the volume for the repo. + +Fixes: #1366 + +.. + https://pagure.io/koji/issue/1366 + volume option for dist-repo + +mikem: When the rpms live on a different volume, it's very inefficient to generate a dist repo on the main volume (because rpms will have to be copied instead of linked). This change allows the user to specify the volume for the repo. + +PR #1327 + + + +**delete_build: handle results of lazy build_references call** + +| PR: https://pagure.io/koji/pull-request/1442 + +Fixes: https://pagure.io/koji/issue/1441 + +.. + https://pagure.io/koji/issue/1441 + delete_build does not handle results of lazy build_references call + +The build references check was recently adjusted to be lazy, but delete_build errors when fields are missing in the return. E.g. + +``` +WARNING:koji.xmlrpc:Traceback (most recent call last): + File "/usr/share/koji-hub/kojixmlrpc.py", line 228, in _wrap_handler + response = handler(environ) + File "/usr/share/koji-hub/kojixmlrpc.py", line 271, in handle_rpc + return self._dispatch(method, params) + File "/usr/share/koji-hub/kojixmlrpc.py", line 308, in _dispatch + ret = koji.util.call_with_argcheck(func, params, opts) + File "/usr/lib/python2.7/site-packages/koji/util.py", line 216, in call_with_argcheck + return func(*args, **kwargs) + File "/usr/share/koji-hub/kojihub.py", line 7215, in delete_build + if refs['archives']: +KeyError: 'archives' +``` + + + +**add --show-channels listing to list-hosts** + +| PR: https://pagure.io/koji/pull-request/1425 + +Fixes: https://pagure.io/koji/issue/1424 + +.. + https://pagure.io/koji/issue/1424 + Add channels to list-hosts + +Add new option --show-channels for optional list of all channels host is subscribed to. + + + +**py2.6 compatibility fix** + +| PR: https://pagure.io/koji/pull-request/1432 + +Python 2.6 doesn't support context manager for GzipFile. Revert to +original behaviour. + +Fixes: https://pagure.io/koji/issue/1431 + +.. + https://pagure.io/koji/issue/1431 + koji-builder-1.17.0 not compatible with python 2.6 + +When running kojid on a python2.6 system you receive the following message: + + + +Traceback (most recent call last): + File "/usr/lib/python2.6/site-packages/koji/daemon.py", line 1295, in runTask + response = (handler.run(),) + File "/usr/lib/python2.6/site-packages/koji/tasks.py", line 311, in run + return koji.util.call_with_argcheck(self.handler, self.params, self.opts) + File "/usr/lib/python2.6/site-packages/koji/util.py", line 263, in call_with_argcheck + return func(*args, **kwargs) + File "/usr/sbin/kojid", line 1302, in handler + broot.init() + File "/usr/sbin/kojid", line 544, in init + self.session.host.setBuildRootList(self.id,self.getPackageList()) + File "/usr/sbin/kojid", line 633, in getPackageList + self.markExternalRPMs(ret) + File "/usr/sbin/kojid", line 764, in markExternalRPMs + with GzipFile(fileobj=fo, mode='r') as fo2: +AttributeError: GzipFile instance has no attribute '__exit__' + +Alas, python2.6 does not have the fix that permits using GzipFile as a 'with' object. + +https://pagure.io/koji/blob/master/f/builder/kojid#_768 + + + +**hub: fix check_fields and duplicated parent_id in _writeInheritanceData** + +| PR: https://pagure.io/koji/pull-request/1434 + +fixes: #1433 +fixes: #1435 + +.. + https://pagure.io/koji/issue/1435 + changes for writeInheritanceData should not contain duplicated parent_id + +duplicated parent_ids will cause inconsistency like: +``` +$ koji call getInheritanceData 48 +[] +$ koji call setInheritanceData 48 --kwargs "{'data':[{'parent_id': 350, 'delete link': True, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''},{'parent_id': 350, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''}]}" +None +$ koji call getInheritanceData 48 +[{'child_id': 48, + 'intransitive': False, + 'maxdepth': None, + 'name': 'test-parent-tag', + 'noconfig': False, + 'parent_id': 350, + 'pkg_filter': '', + 'priority': 10}] +$ koji call setInheritanceData 48 --kwargs "{'data':[{'parent_id': 350, 'delete link': True, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''},{'parent_id': 350, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''}]}" +None +$ koji call getInheritanceData 48 +[] +``` + +.. + https://pagure.io/koji/issue/1433 + writeInheritanceData checks wrong field for "delete link" + +`fields` should be `check_fields` in this case + + + + +**fix table name in build_references query** + +| PR: https://pagure.io/koji/pull-request/1437 + +Fixes: https://pagure.io/koji/issue/1436 + +.. + https://pagure.io/koji/issue/1436 + buildReferences fails for non-rpm builds + +There is a typo in the "most recent use" check in `build_references()` that results in an error like the following: + + + +**build_srpm: Wait until after running the sources command to check for alt_sources_dir** + +| PR: https://pagure.io/koji/pull-request/1410 + +In the RPM layout, it's possible that the SOURCES directory might be +completely empty save for the lookaside payload. In this case a SOURCES +directory wouldn't exist in SCM, and wouldn't be created until the +sources command is run. + +Moving this code block down lets us run the sources command before we try to decide if we should look for the RPM layout or not. + + + +**display task durations in webui** + +| PR: https://pagure.io/koji/pull-request/1383 + +Fixes: https://pagure.io/koji/issue/1382 + +.. + https://pagure.io/koji/issue/1382 + [RFE] taskinfo page can show task durations + +It could be useful to see duration on taskinfo page instead of computing them in the head over and over again. + + + +**rollback errors in multiCall** + +| PR: https://pagure.io/koji/pull-request/1358 + +Fixes: https://pagure.io/koji/issue/1357 + +.. + https://pagure.io/koji/issue/1357 + errors in multicall can result in partial db changes + +The multicall handler catches errors from individual calls and returns the error in the result rather than re-raising it. This sidesteps Koji's normal behavior of rolling back the transaction if an uncaught error occurs in the call. + +This is unlikely to be a problem for most calls, but there are possible cases where a call could make multiple updates, hit an error partway through, and leave those partial updates in place. + + + + + + + + + + +**fix mapping iteration in getFullInheritance** + +| PR: https://pagure.io/koji/pull-request/1406 + +Fixes: https://pagure.io/koji/issue/1405 + +.. + https://pagure.io/koji/issue/1405 + CLI command koji list-tag-inheritance --stop=tag-52zov tag-mjckr returns none + +** _Steps to reproduce: _** +~~~~ + koji list-tag-inheritance --stop=tag-52zov tag-mjckr +~~~~ + + +** _Current Output: _** +~~~~ +koji: Fault: : dictionary changed size during iteration">\ntag-mjckr (21)\n') + +~~~~ + + +**_Expected Output:_** +~~~~ +Show parents/children up to this tag +~~~~ + + + + +**kojid: Download only 'origin'** + +| PR: https://pagure.io/koji/pull-request/1398 + +We have pretty slow connection from s390x koji which helped to uncover +this part. Kojid downloads all files from repomd.xml (incl. filelists) +which is really big. What we really want is just 'origin' (used by Koji +only). + +Signed-off-by: Igor Gnatenko + + + +**Check CLI arguments for enable/disable host** + +| PR: https://pagure.io/koji/pull-request/1365 + +Fixes: https://pagure.io/koji/issue/1364 + +.. + https://pagure.io/koji/issue/1364 + enable-host, disable-host without parameters returns none + +** _Steps to reproduce: _** +~~~~ + koji enable-host + koji disable-host +~~~~ + + +** _Current Output: _** +~~~~ +None +~~~~ + + +**_Expected Output:_** +~~~~ +Usage: koji enable-host... +Usage: koji disable-host... +~~~~ + + + + +**CLI list-channels sorted output** + +| PR: https://pagure.io/koji/pull-request/1390 + +None + + + +**block_pkglist compatibility fix** + +| PR: https://pagure.io/koji/pull-request/1389 + +On older hubs --force is not supported, so CLI will fail on unknown +parameter. This use force option only if it is explicitly required. + +Fixes: https://pagure.io/koji/issue/1388 + +.. + https://pagure.io/koji/issue/1388 + koji-1.17.0-5.fc31 client with 1.16.1 server block doesnt work + +➜ epel7 git:(epel7) koji block-pkg epel7 libdnf +2019-04-05 13:50:26,421 [ERROR] koji: ParameterError: pkglist_block() got an unexpected keyword argument 'force' + +I'm not sure if this is expected or not, but when using the newest client against an older server (koji.fedoraproject.org) block doesn't seem to work from cli. + + + + + + +**scale task_avail_delay based on bin rank** + +| PR: https://pagure.io/koji/pull-request/1386 + +Currently task allocation in Koji is decentralized. The builders pick their next task from a list. The system prefers builders with higher available capacity via the algorithm that the builders use. For a given task, they look at the set of other ready builders for the given channel-arch bin. If the host is below the median, it will not take that task until a waiting period (`task_avail_delay`) has passed. This delay gives higher capacity hosts more of a chance to claim the task. + +Unfortunately, if the set of hosts is very heterogeneous in capacity, the largest capacity hosts might not get used as much as they should because this algorithm does not distinguish any more finely than above/below the median. + +This change generalizes the `task_avail_delay` behavior to scale with the rank of the host within the channel-arch bin. The hosts with highest capacity will take the task immediately, while hosts lower down will have a delay proportional to their rank. We calculate rank as a float between 0.0 and 1.0 and use that as a multiplier for the delay. + +The end result will be that hosts with higher available capacity will be more likely to claim a task, resulting in better utilization of the highest capacity hosts. + + + +**Use createrepo_update even for first repo run** + +| PR: https://pagure.io/koji/pull-request/1363 + +createrepo_update is currently reusing only old repos from same tag. +Nevertheless, for first newRepo there is no old data, but there is a +high chance, that we inherit something. This inherited repo can be used +also for significant speedup. + +Fixes: https://pagure.io/koji/issue/1354 + +.. + https://pagure.io/koji/issue/1354 + [RFE] use createrepo_c --update on new repos (when possible) + +Currently, --update is only used when doing a repo regeneration, not the first time that a repo is generated. + +Let's say that you create the following: +tag: foo-build (inherits from f30-build) + +with a corresponding target. In this case, when generating the repo for foo-build, it would be possible to use --update pointing at f30-build and that would significantly speed up the process. + + + + + + + +**honor mock.package_manager tag setting in mock-config cli** + +| PR: https://pagure.io/koji/pull-request/1374 + +Fixes: #1167 +Fixes: #339 + +This is more of a short term fix for this, but it does the job + +.. + https://pagure.io/koji/issue/1167 + mock-config does not honor settings in extra data + +The config returned by `koji mock-config` does not specify correct package manager even if the tag has it set in the extra data. + +``` +$ koji call getBuildConfig f29-build +{'arches': 'armv7hl i686 x86_64 aarch64 ppc64le s390x', + 'extra': {'mock.package_manager': 'dnf'}, + 'id': 3428, + 'locked': False, + 'maven_include_all': False, + 'maven_support': False, + 'name': 'f29-build', + 'perm': None, + 'perm_id': None} +$ koji mock-config --tag f29-build --arch x86_64 | grep package_manager +$ +``` + +For the CLI command this is definitely a bug, as there's no other way for caller to specify it. + +It would be nice if `koji.genMockConfig` honored the setting as well or there was some other API that would simplify it. Right now to get proper config a user needs to call `getBuildConfig` and then `genMockConfig` ([example](https://pagure.io/rpkg/pull-request/396#request_diff)). This leaks a lot of details. + +.. + https://pagure.io/koji/issue/339 + mock build hangs with f26 mock config + +Today, I used ``koji`` to generate a f26 mock config file + +``` +koji mock-config --tag f26-build --arch=x86_64 --topurl=http://kojipkgs.fedoraproject.org/ -o f26-x86_64.cfg +``` + +and met an issue that following line does not appear in the config as well as repos + +``` +config_opts['package_manager'] = 'dnf' +``` + +BTW, after I added this line manually, + +``mockbuild`` hangs at step + +``` +Start: dnf install +``` + + + + +**Support tilde in search** + +| PR: https://pagure.io/koji/pull-request/1297 + +Fixes https://pagure.io/koji/issue/1294 + +Signed-off-by: Miro Hrončok + +.. + https://pagure.io/koji/issue/1294 + Cannot search build with tilde + +I cannot search this valid build in Koji: **python38-3.8.0~a2-1.fc29** + +https://koji.fedoraproject.org/koji/search?terms=python38-3.8.0~a2-1.fc29&type=build&match=glob + +Bodhi uses this link in https://bodhi.fedoraproject.org/updates/python38-3.8.0~a2-1.fc29 cc @bowlofeggs + +The error is: + +> Search terms may contain only these characters: a-zA-Z0-9 @.,_/\()%+-*?|[]^$ + +I poprose to **add `~` to the list**, as this is completely valid: https://docs.fedoraproject.org/en-US/packaging-guidelines/Versioning/#_versioning_prereleases_with_tilde + + + + + +**kojira: fix iteration over repos in py3** + +| PR: https://pagure.io/koji/pull-request/1356 + +Multiple threads loop over this data, which changes. These loops +need to use a copy. In py2 .keys() and .values() are copied data, +but in py3 they are references to the dict data. + +Fixes: #1355 + +.. + https://pagure.io/koji/issue/1355 + kojira: dictionary changed size error in updateRepos() + +Under python3, kojira seems to hit the following pretty easily. + +``` +Traceback (most recent call last): + File "/usr/sbin/kojira", line 760, in main + repomgr.updateRepos() + File "/usr/sbin/kojira", line 520, in updateRepos + for repo in self.repos.values(): +RuntimeError: dictionary changed size during iteration +``` + + + + +**Fix hub startup handling** + +| PR: https://pagure.io/koji/pull-request/1347 + +Fixes: #875 + +In some environments, module loading can break after restarts. Here we take two steps to prevent that: + +* disable mod_wsgi auto reloading +* use a thread lock for the server setup that happens on first call + +.. + https://pagure.io/koji/issue/875 + hub plugins appear to break hub on code updates + +In the past, we were able to apply hub changes with little or no downtime. + +- upgrade koji packages +- restart httpd + +Of late, we have been seeing errors pop up as soon as the packages update. They look like this: + +``` +2018-04-04 15:00:49,166 [ERROR] m=None u=None p=97080 r=?:? koji.plugins: Traceback (most recent call last): + File "/usr/share/koji-hub/kojixmlrpc.py", line 496, in load_plugins + tracker.load(name) + File "/usr/lib/python2.6/site-packages/koji/plugin.py", line 75, in load + raise koji.PluginError('module name conflict: %s' % mod_name) + PluginError: module name conflict: _koji_plugin__runroot_hub +``` + +With similar errors for other configured hub plugins. + +The problem goes away after an httpd restart. We only load plugins on the first call for each http process, but I guess maybe something is persisting in the module namespace somehow? Or our firstcall check is flawed? + + + +**Rely on ozif_enabled switch in BaseImageTask** + +| PR: https://pagure.io/koji/pull-request/1346 + +Fixes: https://pagure.io/koji/issue/1345 + +.. + https://pagure.io/koji/issue/1345 + Missed ImageFactory detection + +`BaseImageTask` uses IF, but it is not checking `ozif_enabled` flag resulting in: + + ',) + + + +**add .tgz to list of tar's possible extensions** + +| PR: https://pagure.io/koji/pull-request/1344 + +Fixes: https://pagure.io/koji/issue/1343 + +.. + https://pagure.io/koji/issue/1343 + archive extensions for tarball miss .tgz + +.tgz is still sometimes used, let's add it to default set + + + + +**Remove 'keepalive' option** + +| PR: https://pagure.io/koji/pull-request/1277 + +keepalive is not used anymore anywhere in koji + +Fixes: https://pagure.io/koji/issue/1239 + +.. + https://pagure.io/koji/issue/1239 + Deprecate keepalive + +`keepalive` is allowed in config files, while it is not used for anything. Let's deprecate it and remove in some future version. + + + + +**minor gc optimizations** + +| PR: https://pagure.io/koji/pull-request/1337 + +An attempt to make gc a little faster without too much refactor. + + + From 2754228818a4dddcbaff0ea5c941d8c77dbcf28f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 5/14] more relnotes updates --- diff --git a/docs/source/release_notes_1.18.rst b/docs/source/release_notes_1.18.rst index 1da66fc..e436cbd 100644 --- a/docs/source/release_notes_1.18.rst +++ b/docs/source/release_notes_1.18.rst @@ -17,7 +17,7 @@ Security Fixes Client Changes -------------- -**cli: add option for custom cert location** +**Add option for custom cert location** | PR: https://pagure.io/koji/pull-request/1253 @@ -25,7 +25,7 @@ The CLI no has an option for setting a custom SSL certificate, similar to the options for Kerberos authentication. -**cli: load plugins from ~/.koji/plugins** +**Load client plugins from ~/.koji/plugins** | PR: https://pagure.io/koji/pull-request/892 @@ -34,133 +34,54 @@ This change allows users to load their own cli plugins from ``~/.koji/plugins`` or from another location by using the ``plugin_paths`` setting. +**Show load/capacity in list-channels** -Library Changes ---------------- - - -Web UI Changes --------------- - - - -Builder Changes ---------------- - -**use RawConfigParser for kojid** - -| PR: https://pagure.io/koji/pull-request/1544 - -The use of percent signs is common in ``kojid.conf`` because of the -``host_principal_format`` setting. -This causes an error in python3 if ``SafeConfigParser`` is used, so we use -``RawConfigParser`` instead. - - -**handle bare merge mode** - -| PR: https://pagure.io/koji/pull-request/1411 -| PR: https://pagure.io/koji/pull-request/1516 -| PR: https://pagure.io/koji/pull-request/1502 - - -This feature adds a new merge mode for external repos named ``bare``. -This mode is intended for use with modularity. - - - - -System Changes --------------- - - - -**API for reserving NVRs for content generators** - -| PR: https://pagure.io/koji/pull-request/1464 - -Fixes: https://pagure.io/koji/issue/1463 - -.. - https://pagure.io/koji/issue/1463 - [RFE] Predeclare nvr for content generators - -This feature allows content generators to reserve NVRs earlier in the build -process similar to builds performed by ``kojid``. The NVR is reserved by -calling ``CGInitBuild()`` and finalized by the ``CGImport()`` call. - - - -**Add support for tag/target macros for Mageia** - -| PR: https://pagure.io/koji/pull-request/898 - -This feature allows setting rpm macros via the tag extra field. These macros -will be added to the mock configuration for the buildroot. The system -looks for extra values of the form ``rpm.macro.NAME``. - -For example, to set the dist tag for a given tag, you could use a command like: - -:: - - $ koji edit-tag f30-build -x rpm.macro.dist=MYDISTTAG - - - -**set module_hotfixes=1 in yum.conf via tag config** - -| PR: https://pagure.io/koji/pull-request/1524 +| PR: https://pagure.io/koji/pull-request/1449 -Koji now handles the field ``mock.yum.module_hotfixes`` in the tag extra. -When set, kojid will set ``module_hotfixes=0/1`` in the yum portion of the -mock configuration for a buildroot. +The ``list-channels`` display has been expanded to show overall totals for load +and capacity. -**Allow users to opt out of notifications** +**Sorted output for list-channels** -| PR: https://pagure.io/koji/pull-request/1417 +| PR: https://pagure.io/koji/pull-request/1390 -This feature lets users opt out of notifications that they would otherwise -automatically recieve, such as build and tag notifications for: +The channel list is now sorted -- the build owner (the user who submitted the build) -- the package owner within the given tag -These opt-outs are user controlled and can be managed with the new -``block-notification`` and ``unblock-notificiation`` commands. +**Allow taginfo cli to use tag IDs** +| PR: https://pagure.io/koji/pull-request/1476 +The ``taginfo`` command can now accept a numeric tag id on the command line. -**Allow hub policy to match version and release** -| PR: https://pagure.io/koji/pull-request/1513 +**Add --show-channels option to list-hosts** +| PR: https://pagure.io/koji/pull-request/1425 -This feature adds new policy tests to match ``version`` and ``release``. -This tests are glob pattern matches. +The ``list-hosts`` command can now display channel subscriptions. -**Rebuild SPMS before building** +**Remove merge option from edit-external-repo** -| PR: https://pagure.io/koji/pull-request/1462 +| PR: https://pagure.io/koji/pull-request/1499 -For rpm builds from an uploaded srpm, Koji will now rebuild the srpm in the -build environment first. -This ensures that the NVR is correct for the resulting build. +This option was mistakenly added to the command and never did anything. +It is gone now. -The old behavior can be requested by setting ``rebuild_srpm=False`` in the tag -extra data for the build tag in question. +**Honor mock.package_manager tag setting in mock-config cli** +| PR: https://pagure.io/koji/pull-request/1374 +The ``mock-config`` command will now honor this setting just as ``kojid`` does. -**remove merge option from edit-external-repo** -| PR: https://pagure.io/koji/pull-request/1499 -This option was mistakenly added to the command and never did anything. -It is gone now. +Library Changes +--------------- **New multicall interface** @@ -186,19 +107,10 @@ document. -**New 'buildtype' test for policies** - -| PR: https://pagure.io/koji/pull-request/1415 - - -Koji added btypes in version 1.11 along with content generators. -Now, all builds have one or more btypes. - -This change allows policies to check the btype value using the ``buildtype`` test. - - +Web UI Changes +-------------- -**retain old search pattern in web ui** +**Retain old search pattern in web ui** | PR: https://pagure.io/koji/pull-request/1258 @@ -207,721 +119,243 @@ current search pre-filled. This makes it easier for users to refine their searches. +**Display task durations in webui** +| PR: https://pagure.io/koji/pull-request/1383 -**introduce host-admin permission + docs** - -| PR: https://pagure.io/koji/pull-request/1454 - - -I'm filing this as first of series to splitting admin permission to more granular ones. Adding some docs about permission system and `host`, `tag` and `target` permissions. - - - -**createrepo_c is used by default now** - -| PR: https://pagure.io/koji/pull-request/1278 - -Fixes: https://pagure.io/koji/issue/716 - -.. - https://pagure.io/koji/issue/716 - /usr/libexec/kojid/mergerepos ignores xml:base in location - -If you add an external repo to a build tag, and the external repo uses the same xml:base trickery that Koji itself uses (that is, if the external repo you are pointing it is produced by another Koji instance for a tag which itself has an external repo configure), the xml:base attribute is lost by /usr/libexec/kojid/mergerepos and the resulting merged repodata ends up with incorrect URLs. - -To give a hypothetical example, imagine your Koji is producing a repo from a build tag at: - -http://thirdpartykoji.example.com/kojifiles/repos/beaker-harness-rhel-8-build/latest/x86_64/repodata/ - -It uses an external repo to pull packages from another Koji instance at: - -http://download.bigcorp.com/brewroot/repos/rhel-8.0-build/latest/x86_64/repodata/ - -But that repo itself is pulling packages from an external repo: - -https://kojipkgs.fedoraproject.org/repos/module-bootstrap-rawhide/latest/x86_64/repodata/ - -The download.bigcorp.com repodata will refer to the remote packages using xml:base, like this: - - - -Everything is fine so far. But now, when mergerepos runs on thirdpartykoji.example.com, I expect it to preserve the same so that the packages can be correctly downloaded no matter how many layers of external repos are involved. However mergerepos on thirdpartykoji.example.com instead produces this: - - - -which fails to download because the package is not there. The original value of xml:base has been lost. - -The end result is build failures with errors like this in mock_output.log as yum/dnf tries to download the packages from the wrong URL: - - http://download.bigcorp.com/brewroot/repos/rhel-8.0-build/latest/i386/audit-libs-2.7.7-5.fc27.i686.rpm: [Errno 14] HTTP Error 404 - Not Found - Trying other mirror. - - - -**show load/capacity in list-channels** - -| PR: https://pagure.io/koji/pull-request/1449 - -Fixes: https://pagure.io/koji/issue/1448 - -.. - https://pagure.io/koji/issue/1448 - [RFE] show channel load/capacity in list-channels - -list-channels can show overall number for load/capacity - - - -**Allow taginfo cli to use tag IDs; fixed Inheritance printing bug** - -| PR: https://pagure.io/koji/pull-request/1476 - -It would be useful to be able to use the koji cli's taginfo with tag IDs and not just the tag name since the python library allows for this. Also there is a bug where Inheritance always uses the value of the last tag queried for. +The ``taskinfo`` page in the web ui now shows task durations in addition to +timestamps. -Old behavior (ID bug): -``` -❯❯❯ koji taginfo 6438 -No such tag: 6438 -``` -Old behavior (Inheritance bug): -``` -❯❯❯ koji taginfo rawhide f31 -Tag: rawhide [197] -Arches: aarch64 armv7hl i686 ppc64 ppc64le s390x x86_64 -Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build -LOCKED -Tag options: -Inheritance: - <<<<< inheritance missing/incorrect -Tag: f31 [6438] -Arches: None -Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build -Required permission: 'autosign' -Tag options: - mock.new_chroot : 0 - mock.package_manager : 'dnf' -Inheritance: +Builder Changes +--------------- -``` -New behavior: -``` -❯❯❯ koji taginfo rawhide 6438 -Tag: rawhide [197] -Arches: aarch64 armv7hl i686 ppc64 ppc64le s390x x86_64 -Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build -LOCKED -Tag options: -Inheritance: - 0 .... f31 [6438] +**Rebuild SPMS before building** -Tag: f31 [6438] -Arches: None -Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build -Required permission: 'autosign' -Tag options: - mock.new_chroot : 0 - mock.package_manager : 'dnf' -Inheritance: -``` +| PR: https://pagure.io/koji/pull-request/1462 -.. - https://pagure.io/koji/issue/1485 - taginfo shows wrong inheritance when multiple tags given +For rpm builds from an uploaded srpm, Koji will now rebuild the srpm in the +build environment first. +This ensures that the NVR is correct for the resulting build. -When multiple tags are requested, the inheritance data for the final tag is shown for all tags. +The old behavior can be requested by setting ``rebuild_srpm=False`` in the tag +extra data for the build tag in question. -``` -$ koji taginfo f31 f28-build -Tag: f31 [6438] -Arches: None -Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build -Required permission: 'autosign' -Tag options: - mock.new_chroot : 0 - mock.package_manager : 'dnf' -Inheritance: - 0 .... f28-override [1922] -Tag: f28-build [1928] -Arches: armv7hl i686 x86_64 aarch64 ppc64 ppc64le s390x -Groups: appliance-build, build, livecd-build, livemedia-build, srpm-build -Tag options: -Inheritance: - 0 .... f28-override [1922] -``` +**Handle bare merge mode** -Here we see `f28`'s inheritance for both tags (in reality, `f31` has no inheritance). +| PR: https://pagure.io/koji/pull-request/1411 +| PR: https://pagure.io/koji/pull-request/1516 +| PR: https://pagure.io/koji/pull-request/1502 +This feature adds a new merge mode for external repos named ``bare``. +This mode is intended for use with modularity. +Use of this mode requires createrepo_c version 0.14.0 or later on the builders +that handle the createrepo tasks. +**User createrepo_c by default** -**deprecate BuildRoot.uploadDir method** +| PR: https://pagure.io/koji/pull-request/1278 -| PR: https://pagure.io/koji/pull-request/1456 -Fixes: https://pagure.io/koji/issue/839 +The ``use_createrepo_c`` configuration option for ``kojid`` now defaults to True. -.. - https://pagure.io/koji/issue/839 - deprecate BuildRoot.uploadDir() -Noticed while working on a patch that this method is not used +**Use createrepo --update option even for first repo run** +| PR: https://pagure.io/koji/pull-request/1363 +If there is no older repo for a tag, Koji will now attempt to find +a related repo to use ``createrepo --update`` with. +This will speed up first-time repo generations for tags that +predominantly inherit their content from another build tag. -**check existence of tag_id in getInheritanceData** -| PR: https://pagure.io/koji/pull-request/1461 +**Scale task_avail_delay based on bin rank** -Fixes: https://pagure.io/koji/issue/1460 +| PR: https://pagure.io/koji/pull-request/1386 -.. - https://pagure.io/koji/issue/1460 - getInheritanceData API call should raise GenericError exception for non existing tag ID +This is an adjustment to Koji's decentralized scheduling algorithm. +It should result in better utilization of host capacity, particularly when +a channel has hosts that are very heterogeneous in capacity. -*Steps to Reproduce:* -Run a getInheritanceData(non_exist_tag_ID) +The meaning of the ``task_avail_delay`` setting is different now. +Within a channel-arch bin, the hosts with highest capacity will take the task +immediately, while hosts lower down will have a delay proportional to their +rank. +The "rank" here is a float between 0.0 and 1.0 used as a multiplier. +So ``task_avail_delay`` is the maximum time that the any host will wait to +take a task. -*Actual result:* +This means hosts with higher available capacity will be more likely to claim a +task, resulting in better utilization of the highest capacity hosts. -[] -*Expected result:* +**Use RawConfigParser for kojid** -GenericError exception should be raised +| PR: https://pagure.io/koji/pull-request/1544 +The use of percent signs is common in ``kojid.conf`` because of the +``host_principal_format`` setting. +This causes an error in python3 if ``SafeConfigParser`` is used, so we use +``RawConfigParser`` instead. +System Changes +-------------- -**Allow generating separate src repo for build repos** -| PR: https://pagure.io/koji/pull-request/1273 +**API for reserving NVRs for content generators** -Fixes #1266 +| PR: https://pagure.io/koji/pull-request/1464 -Currently Koji has an option that can be used to include source rpms in each of generated arch repos. Howewer it makes repositories significantly bigger and slower to generate. Source metadata is duplicated across all arch repos. Increased metadata size makes builds slower. +This feature allows content generators to reserve NVRs earlier in the build +process similar to builds performed by ``kojid``. The NVR is reserved by +calling ``CGInitBuild()`` and finalized by the ``CGImport()`` call. -This pull request takes a slightly different approach - it makes it possible to generate build repos with source rpms in separate repos. Such repos don't change size of each arch repo and can be generated faster - generation is done in a separate createrepo task that can run on separate host. For example: - newRepo - ├ createrepo (src) - ├ createrepo (aarch64) - ├ createrepo (ppc64le) - └ createrepo (x86_64) -CC @ignatenkobrain +**Per-tag configuration of rpm macros** -.. - https://pagure.io/koji/issue/1266 - RFE: --with-separate-src for build-repos +| PR: https://pagure.io/koji/pull-request/898 -In Fedora we want to have src in build repos, but in separate repo so the size of metadata for builds won't grow. +This feature allows setting rpm macros via the tag extra field. These macros +will be added to the mock configuration for the buildroot. The system +looks for extra values of the form ``rpm.macro.NAME``. +For example, to set the dist tag for a given tag, you could use a command like: +:: + $ koji edit-tag f30-build -x rpm.macro.dist=MYDISTTAG -**add strict option to getTaskChildren** -| PR: https://pagure.io/koji/pull-request/1256 +**Per-tag configuration for module_hotfixes setting** -+ moving to QueryProcessor of used calls +| PR: https://pagure.io/koji/pull-request/1524 -.. - https://pagure.io/koji/issue/1199 - API Call getTaskChildren call should return GenericError for non existing taskID +Koji now handles the field ``mock.yum.module_hotfixes`` in the tag extra. +When set, kojid will set ``module_hotfixes=0/1`` in the yum portion of the +mock configuration for a buildroot. -*Steps to Reproduce:* -Run getTaskChildren(non_existing_id) +**Allow users to opt out of notifications** -*Actual result:* +| PR: https://pagure.io/koji/pull-request/1417 -[] +This feature lets users opt out of notifications that they would otherwise +automatically recieve, such as build and tag notifications for: -*Expected result:* +- the build owner (the user who submitted the build) +- the package owner within the given tag -Should be returning an error message that task id does not exist. +These opt-outs are user controlled and can be managed with the new +``block-notification`` and ``unblock-notificiation`` commands. -I suggest add a strict option for the getTaskChildren API call. +**Allow hub policy to match version and release** +| PR: https://pagure.io/koji/pull-request/1513 -**fail runroot task on non-existing tag** +This feature adds new policy tests to match ``version`` and ``release``. +This tests are glob pattern matches. -| PR: https://pagure.io/koji/pull-request/1257 -Fixes: https://pagure.io/koji/issue/1139 +**New 'buildtype' test for policies** -.. - https://pagure.io/koji/issue/1139 - runroot API call should raise GenericError exception for non existing tag +| PR: https://pagure.io/koji/pull-request/1415 -*Steps to Reproduce:* -tag = 'non-existing-tag' -arch 'x86_64' -Run runroot(tag, arch, ['echo', 'hello', 'world'], ) +Koji added btypes in version 1.11 along with content generators. +Now, all builds have one or more btypes. -*Actual result:* +This change allows policies to check the btype value using the ``buildtype`` test. -API call doesn't return some message -Web interface returns this after the click to runroot task: -Error -An error has occurred while processing your request. -IndexError: list index out of range -Full tracebacks disabled -*Expected result:* +**More granular admin permissions** -Should be returning an error message that tag is not existing and on web interface should be normal task info with the failed state. -[![Screenshot_from_2018-10-31_09-50-46.png](/koji/issue/raw/files/b1dde9720c1fac719c568be125886ae087ec391a9911ed2e30fc3c228d7c452d-Screenshot_from_2018-10-31_09-50-46.png)](/koji/issue/raw/files/b1dde9720c1fac719c568be125886ae087ec391a9911ed2e30fc3c228d7c452d-Screenshot_from_2018-10-31_09-50-46.png) +| PR: https://pagure.io/koji/pull-request/1454 +A number of actions that were previously admin-only are now governed by +separate permissions: + ``host`` + This permission governs most host management operations, such as + adding, editing, enabling/disabling, and restarting. + ``tag`` + This permission governs adding, editing, and deleting tags. -**check architecture names for mistakes** + ``target`` + This permission governs adding, editing, and deleting targets. -| PR: https://pagure.io/koji/pull-request/1272 -Fixes: https://pagure.io/koji/issue/1237 -.. - https://pagure.io/koji/issue/1237 - Architectures field in build target uses 2 different separators +**Option to generate separate source repo** -Koji returns inconsistent data for architectures from build target, because both space and comma separators are allowed. +| PR: https://pagure.io/koji/pull-request/1273 -Could be possible to unify separators to space character only (at least in API responses)? +The (non-dist) yum repos that Koji generates for building normally don't +include srpms. +An old option allowed them to be included in some cases, but they were simply +added to each repo. +Newer options have been added that instruct Koji to include them as a separate +src repo. -Ideally it would be great to get same order of the architectures no matter in which order they have been provided (e.g. sort them alphabetically). +In the cli, the ``regen-repo`` command now accepts a ``--separate-source`` +option that triggers this behavior. +In ``kojira``, the ``separate_source_tags`` option is a list of tag patterns. +Build tags that match any of these patterns will have their repos generated +with a separate src repo. -**volume option for dist-repo** +**Add volume option for dist-repo** | PR: https://pagure.io/koji/pull-request/1327 -When the rpms live on a different volume, it's very inefficient to generate a dist repo on the main volume (because rpms will have to be copied instead of linked). This change allows the user to specify the volume for the repo. - -Fixes: #1366 - -.. - https://pagure.io/koji/issue/1366 - volume option for dist-repo +Dist repos can now be generated on volumes other than the main one. +Use the ``--volume`` option to the ``dist-repo`` command to do so. -mikem: When the rpms live on a different volume, it's very inefficient to generate a dist repo on the main volume (because rpms will have to be copied instead of linked). This change allows the user to specify the volume for the repo. +Generally you want the repo to be on the same volume as the rpms it will +contain. +Dist repos hard link (same volume) or copy (different volume) their rpms into +place. +Using the appropriate volume can drastically improve the efficiency, both in +generation time and space consumption. -PR #1327 +**Minor gc optimizations** - -**delete_build: handle results of lazy build_references call** - +| PR: https://pagure.io/koji/pull-request/1337 | PR: https://pagure.io/koji/pull-request/1442 - -Fixes: https://pagure.io/koji/issue/1441 - -.. - https://pagure.io/koji/issue/1441 - delete_build does not handle results of lazy build_references call - -The build references check was recently adjusted to be lazy, but delete_build errors when fields are missing in the return. E.g. - -``` -WARNING:koji.xmlrpc:Traceback (most recent call last): - File "/usr/share/koji-hub/kojixmlrpc.py", line 228, in _wrap_handler - response = handler(environ) - File "/usr/share/koji-hub/kojixmlrpc.py", line 271, in handle_rpc - return self._dispatch(method, params) - File "/usr/share/koji-hub/kojixmlrpc.py", line 308, in _dispatch - ret = koji.util.call_with_argcheck(func, params, opts) - File "/usr/lib/python2.7/site-packages/koji/util.py", line 216, in call_with_argcheck - return func(*args, **kwargs) - File "/usr/share/koji-hub/kojihub.py", line 7215, in delete_build - if refs['archives']: -KeyError: 'archives' -``` - - - -**add --show-channels listing to list-hosts** - -| PR: https://pagure.io/koji/pull-request/1425 - -Fixes: https://pagure.io/koji/issue/1424 - -.. - https://pagure.io/koji/issue/1424 - Add channels to list-hosts - -Add new option --show-channels for optional list of all channels host is subscribed to. - - - -**py2.6 compatibility fix** - -| PR: https://pagure.io/koji/pull-request/1432 - -Python 2.6 doesn't support context manager for GzipFile. Revert to -original behaviour. - -Fixes: https://pagure.io/koji/issue/1431 - -.. - https://pagure.io/koji/issue/1431 - koji-builder-1.17.0 not compatible with python 2.6 - -When running kojid on a python2.6 system you receive the following message: - - - -Traceback (most recent call last): - File "/usr/lib/python2.6/site-packages/koji/daemon.py", line 1295, in runTask - response = (handler.run(),) - File "/usr/lib/python2.6/site-packages/koji/tasks.py", line 311, in run - return koji.util.call_with_argcheck(self.handler, self.params, self.opts) - File "/usr/lib/python2.6/site-packages/koji/util.py", line 263, in call_with_argcheck - return func(*args, **kwargs) - File "/usr/sbin/kojid", line 1302, in handler - broot.init() - File "/usr/sbin/kojid", line 544, in init - self.session.host.setBuildRootList(self.id,self.getPackageList()) - File "/usr/sbin/kojid", line 633, in getPackageList - self.markExternalRPMs(ret) - File "/usr/sbin/kojid", line 764, in markExternalRPMs - with GzipFile(fileobj=fo, mode='r') as fo2: -AttributeError: GzipFile instance has no attribute '__exit__' - -Alas, python2.6 does not have the fix that permits using GzipFile as a 'with' object. - -https://pagure.io/koji/blob/master/f/builder/kojid#_768 - - - -**hub: fix check_fields and duplicated parent_id in _writeInheritanceData** - -| PR: https://pagure.io/koji/pull-request/1434 - -fixes: #1433 -fixes: #1435 - -.. - https://pagure.io/koji/issue/1435 - changes for writeInheritanceData should not contain duplicated parent_id - -duplicated parent_ids will cause inconsistency like: -``` -$ koji call getInheritanceData 48 -[] -$ koji call setInheritanceData 48 --kwargs "{'data':[{'parent_id': 350, 'delete link': True, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''},{'parent_id': 350, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''}]}" -None -$ koji call getInheritanceData 48 -[{'child_id': 48, - 'intransitive': False, - 'maxdepth': None, - 'name': 'test-parent-tag', - 'noconfig': False, - 'parent_id': 350, - 'pkg_filter': '', - 'priority': 10}] -$ koji call setInheritanceData 48 --kwargs "{'data':[{'parent_id': 350, 'delete link': True, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''},{'parent_id': 350, 'priority': 10, 'maxdepth': None, 'intransitive': False, 'noconfig': False, 'pkg_filter': ''}]}" -None -$ koji call getInheritanceData 48 -[] -``` - -.. - https://pagure.io/koji/issue/1433 - writeInheritanceData checks wrong field for "delete link" - -`fields` should be `check_fields` in this case - - - - -**fix table name in build_references query** - | PR: https://pagure.io/koji/pull-request/1437 -Fixes: https://pagure.io/koji/issue/1436 - -.. - https://pagure.io/koji/issue/1436 - buildReferences fails for non-rpm builds - -There is a typo in the "most recent use" check in `build_references()` that results in an error like the following: - - +This change speeds up portions of garbage collection by making the +``build_references`` check lazy by default. -**build_srpm: Wait until after running the sources command to check for alt_sources_dir** -| PR: https://pagure.io/koji/pull-request/1410 -In the RPM layout, it's possible that the SOURCES directory might be -completely empty save for the lookaside payload. In this case a SOURCES -directory wouldn't exist in SCM, and wouldn't be created until the -sources command is run. - -Moving this code block down lets us run the sources command before we try to decide if we should look for the RPM layout or not. - - - -**display task durations in webui** - -| PR: https://pagure.io/koji/pull-request/1383 - -Fixes: https://pagure.io/koji/issue/1382 - -.. - https://pagure.io/koji/issue/1382 - [RFE] taskinfo page can show task durations - -It could be useful to see duration on taskinfo page instead of computing them in the head over and over again. - - - -**rollback errors in multiCall** +**Rollback errors in multiCall** | PR: https://pagure.io/koji/pull-request/1358 -Fixes: https://pagure.io/koji/issue/1357 - -.. - https://pagure.io/koji/issue/1357 - errors in multicall can result in partial db changes - -The multicall handler catches errors from individual calls and returns the error in the result rather than re-raising it. This sidesteps Koji's normal behavior of rolling back the transaction if an uncaught error occurs in the call. - -This is unlikely to be a problem for most calls, but there are possible cases where a call could make multiple updates, hit an error partway through, and leave those partial updates in place. - - - - - - - - - - -**fix mapping iteration in getFullInheritance** - -| PR: https://pagure.io/koji/pull-request/1406 - -Fixes: https://pagure.io/koji/issue/1405 - -.. - https://pagure.io/koji/issue/1405 - CLI command koji list-tag-inheritance --stop=tag-52zov tag-mjckr returns none - -** _Steps to reproduce: _** -~~~~ - koji list-tag-inheritance --stop=tag-52zov tag-mjckr -~~~~ - - -** _Current Output: _** -~~~~ -koji: Fault: : dictionary changed size during iteration">\ntag-mjckr (21)\n') - -~~~~ - - -**_Expected Output:_** -~~~~ -Show parents/children up to this tag -~~~~ - - - - -**kojid: Download only 'origin'** - -| PR: https://pagure.io/koji/pull-request/1398 - -We have pretty slow connection from s390x koji which helped to uncover -this part. Kojid downloads all files from repomd.xml (incl. filelists) -which is really big. What we really want is just 'origin' (used by Koji -only). - -Signed-off-by: Igor Gnatenko - - - -**Check CLI arguments for enable/disable host** - -| PR: https://pagure.io/koji/pull-request/1365 - -Fixes: https://pagure.io/koji/issue/1364 - -.. - https://pagure.io/koji/issue/1364 - enable-host, disable-host without parameters returns none - -** _Steps to reproduce: _** -~~~~ - koji enable-host - koji disable-host -~~~~ - - -** _Current Output: _** -~~~~ -None -~~~~ - - -**_Expected Output:_** -~~~~ -Usage: koji enable-host... -Usage: koji disable-host... -~~~~ - - - - -**CLI list-channels sorted output** - -| PR: https://pagure.io/koji/pull-request/1390 - -None - - - -**block_pkglist compatibility fix** - -| PR: https://pagure.io/koji/pull-request/1389 - -On older hubs --force is not supported, so CLI will fail on unknown -parameter. This use force option only if it is explicitly required. - -Fixes: https://pagure.io/koji/issue/1388 - -.. - https://pagure.io/koji/issue/1388 - koji-1.17.0-5.fc31 client with 1.16.1 server block doesnt work - -➜ epel7 git:(epel7) koji block-pkg epel7 libdnf -2019-04-05 13:50:26,421 [ERROR] koji: ParameterError: pkglist_block() got an unexpected keyword argument 'force' - -I'm not sure if this is expected or not, but when using the newest client against an older server (koji.fedoraproject.org) block doesn't seem to work from cli. - - - - - - -**scale task_avail_delay based on bin rank** - -| PR: https://pagure.io/koji/pull-request/1386 - -Currently task allocation in Koji is decentralized. The builders pick their next task from a list. The system prefers builders with higher available capacity via the algorithm that the builders use. For a given task, they look at the set of other ready builders for the given channel-arch bin. If the host is below the median, it will not take that task until a waiting period (`task_avail_delay`) has passed. This delay gives higher capacity hosts more of a chance to claim the task. - -Unfortunately, if the set of hosts is very heterogeneous in capacity, the largest capacity hosts might not get used as much as they should because this algorithm does not distinguish any more finely than above/below the median. - -This change generalizes the `task_avail_delay` behavior to scale with the rank of the host within the channel-arch bin. The hosts with highest capacity will take the task immediately, while hosts lower down will have a delay proportional to their rank. We calculate rank as a float between 0.0 and 1.0 and use that as a multiplier for the delay. - -The end result will be that hosts with higher available capacity will be more likely to claim a task, resulting in better utilization of the highest capacity hosts. - - - -**Use createrepo_update even for first repo run** - -| PR: https://pagure.io/koji/pull-request/1363 - -createrepo_update is currently reusing only old repos from same tag. -Nevertheless, for first newRepo there is no old data, but there is a -high chance, that we inherit something. This inherited repo can be used -also for significant speedup. - -Fixes: https://pagure.io/koji/issue/1354 - -.. - https://pagure.io/koji/issue/1354 - [RFE] use createrepo_c --update on new repos (when possible) - -Currently, --update is only used when doing a repo regeneration, not the first time that a repo is generated. - -Let's say that you create the following: -tag: foo-build (inherits from f30-build) - -with a corresponding target. In this case, when generating the repo for foo-build, it would be possible to use --update pointing at f30-build and that would significantly speed up the process. - - - - - - - -**honor mock.package_manager tag setting in mock-config cli** - -| PR: https://pagure.io/koji/pull-request/1374 - -Fixes: #1167 -Fixes: #339 - -This is more of a short term fix for this, but it does the job - -.. - https://pagure.io/koji/issue/1167 - mock-config does not honor settings in extra data - -The config returned by `koji mock-config` does not specify correct package manager even if the tag has it set in the extra data. - -``` -$ koji call getBuildConfig f29-build -{'arches': 'armv7hl i686 x86_64 aarch64 ppc64le s390x', - 'extra': {'mock.package_manager': 'dnf'}, - 'id': 3428, - 'locked': False, - 'maven_include_all': False, - 'maven_support': False, - 'name': 'f29-build', - 'perm': None, - 'perm_id': None} -$ koji mock-config --tag f29-build --arch x86_64 | grep package_manager -$ -``` - -For the CLI command this is definitely a bug, as there's no other way for caller to specify it. - -It would be nice if `koji.genMockConfig` honored the setting as well or there was some other API that would simplify it. Right now to get proper config a user needs to call `getBuildConfig` and then `genMockConfig` ([example](https://pagure.io/rpkg/pull-request/396#request_diff)). This leaks a lot of details. - -.. - https://pagure.io/koji/issue/339 - mock build hangs with f26 mock config - -Today, I used ``koji`` to generate a f26 mock config file - -``` -koji mock-config --tag f26-build --arch=x86_64 --topurl=http://kojipkgs.fedoraproject.org/ -o f26-x86_64.cfg -``` - -and met an issue that following line does not appear in the config as well as repos - -``` -config_opts['package_manager'] = 'dnf' -``` - -BTW, after I added this line manually, - -``mockbuild`` hangs at step - -``` -Start: dnf install -``` +If one of the calls in a multicall raises an error, then the transaction will +be rolled back to the start of that call before Koji proceeds to the next call. +This matches the behavior of normal calls more closely. +Multicalls are still handled within single database transaction. @@ -929,133 +363,8 @@ Start: dnf install | PR: https://pagure.io/koji/pull-request/1297 -Fixes https://pagure.io/koji/issue/1294 - -Signed-off-by: Miro Hrončok - -.. - https://pagure.io/koji/issue/1294 - Cannot search build with tilde - -I cannot search this valid build in Koji: **python38-3.8.0~a2-1.fc29** - -https://koji.fedoraproject.org/koji/search?terms=python38-3.8.0~a2-1.fc29&type=build&match=glob - -Bodhi uses this link in https://bodhi.fedoraproject.org/updates/python38-3.8.0~a2-1.fc29 cc @bowlofeggs - -The error is: - -> Search terms may contain only these characters: a-zA-Z0-9 @.,_/\()%+-*?|[]^$ - -I poprose to **add `~` to the list**, as this is completely valid: https://docs.fedoraproject.org/en-US/packaging-guidelines/Versioning/#_versioning_prereleases_with_tilde - - - - - -**kojira: fix iteration over repos in py3** - -| PR: https://pagure.io/koji/pull-request/1356 - -Multiple threads loop over this data, which changes. These loops -need to use a copy. In py2 .keys() and .values() are copied data, -but in py3 they are references to the dict data. - -Fixes: #1355 - -.. - https://pagure.io/koji/issue/1355 - kojira: dictionary changed size error in updateRepos() - -Under python3, kojira seems to hit the following pretty easily. - -``` -Traceback (most recent call last): - File "/usr/sbin/kojira", line 760, in main - repomgr.updateRepos() - File "/usr/sbin/kojira", line 520, in updateRepos - for repo in self.repos.values(): -RuntimeError: dictionary changed size during iteration -``` - - - - -**Fix hub startup handling** - -| PR: https://pagure.io/koji/pull-request/1347 - -Fixes: #875 - -In some environments, module loading can break after restarts. Here we take two steps to prevent that: - -* disable mod_wsgi auto reloading -* use a thread lock for the server setup that happens on first call - -.. - https://pagure.io/koji/issue/875 - hub plugins appear to break hub on code updates - -In the past, we were able to apply hub changes with little or no downtime. - -- upgrade koji packages -- restart httpd - -Of late, we have been seeing errors pop up as soon as the packages update. They look like this: - -``` -2018-04-04 15:00:49,166 [ERROR] m=None u=None p=97080 r=?:? koji.plugins: Traceback (most recent call last): - File "/usr/share/koji-hub/kojixmlrpc.py", line 496, in load_plugins - tracker.load(name) - File "/usr/lib/python2.6/site-packages/koji/plugin.py", line 75, in load - raise koji.PluginError('module name conflict: %s' % mod_name) - PluginError: module name conflict: _koji_plugin__runroot_hub -``` - -With similar errors for other configured hub plugins. - -The problem goes away after an httpd restart. We only load plugins on the first call for each http process, but I guess maybe something is persisting in the module namespace somehow? Or our firstcall check is flawed? - - - -**Rely on ozif_enabled switch in BaseImageTask** - -| PR: https://pagure.io/koji/pull-request/1346 - -Fixes: https://pagure.io/koji/issue/1345 - -.. - https://pagure.io/koji/issue/1345 - Missed ImageFactory detection - -`BaseImageTask` uses IF, but it is not checking `ozif_enabled` flag resulting in: - - ',) - - - -**add .tgz to list of tar's possible extensions** - -| PR: https://pagure.io/koji/pull-request/1344 - -Fixes: https://pagure.io/koji/issue/1343 - -.. - https://pagure.io/koji/issue/1343 - archive extensions for tarball miss .tgz - -.tgz is still sometimes used, let's add it to default set +The tilde character is no longer prohibited in search terms. @@ -1063,24 +372,5 @@ Fixes: https://pagure.io/koji/issue/1343 | PR: https://pagure.io/koji/pull-request/1277 -keepalive is not used anymore anywhere in koji - -Fixes: https://pagure.io/koji/issue/1239 - -.. - https://pagure.io/koji/issue/1239 - Deprecate keepalive - -`keepalive` is allowed in config files, while it is not used for anything. Let's deprecate it and remove in some future version. - - - - -**minor gc optimizations** - -| PR: https://pagure.io/koji/pull-request/1337 - -An attempt to make gc a little faster without too much refactor. - - - +The ``keepalive`` setting is no longer used anywhere in koji. +It has been removed. From 87033e69806d47d6b4ecbe5fe2a34aa87f5859bb Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 6/14] cleanup --- diff --git a/docs/source/release_notes_1.18.rst b/docs/source/release_notes_1.18.rst index e436cbd..d27f218 100644 --- a/docs/source/release_notes_1.18.rst +++ b/docs/source/release_notes_1.18.rst @@ -21,7 +21,7 @@ Client Changes | PR: https://pagure.io/koji/pull-request/1253 -The CLI no has an option for setting a custom SSL certificate, similar to the +The CLI now has an option for setting a custom SSL certificate, similar to the options for Kerberos authentication. @@ -42,13 +42,6 @@ The ``list-channels`` display has been expanded to show overall totals for load and capacity. -**Sorted output for list-channels** - -| PR: https://pagure.io/koji/pull-request/1390 - -The channel list is now sorted - - **Allow taginfo cli to use tag IDs** | PR: https://pagure.io/koji/pull-request/1476 @@ -56,11 +49,12 @@ The channel list is now sorted The ``taginfo`` command can now accept a numeric tag id on the command line. -**Add --show-channels option to list-hosts** +**Add option to show channels in list-hosts** | PR: https://pagure.io/koji/pull-request/1425 -The ``list-hosts`` command can now display channel subscriptions. +The ``list-hosts`` command will now display channel subscriptions if the +``--show-channels`` option is given. **Remove merge option from edit-external-repo** @@ -101,7 +95,7 @@ With this new implementation: * method calls return a `VirtualCall` instance that can later be used to access the result * `MultiCallSession` can be used as a context manager, ensuring that the calls are executed -Usage examples are availble in the :doc:`Writing Koji Code ` +Usage examples can be found in the :doc:`Writing Koji Code ` document. @@ -144,20 +138,6 @@ The old behavior can be requested by setting ``rebuild_srpm=False`` in the tag extra data for the build tag in question. -**Handle bare merge mode** - -| PR: https://pagure.io/koji/pull-request/1411 -| PR: https://pagure.io/koji/pull-request/1516 -| PR: https://pagure.io/koji/pull-request/1502 - - -This feature adds a new merge mode for external repos named ``bare``. -This mode is intended for use with modularity. - -Use of this mode requires createrepo_c version 0.14.0 or later on the builders -that handle the createrepo tasks. - - **User createrepo_c by default** | PR: https://pagure.io/koji/pull-request/1278 @@ -166,7 +146,7 @@ that handle the createrepo tasks. The ``use_createrepo_c`` configuration option for ``kojid`` now defaults to True. -**Use createrepo --update option even for first repo run** +**Use createrepo update option even for first repo run** | PR: https://pagure.io/koji/pull-request/1363 @@ -189,10 +169,10 @@ Within a channel-arch bin, the hosts with highest capacity will take the task immediately, while hosts lower down will have a delay proportional to their rank. The "rank" here is a float between 0.0 and 1.0 used as a multiplier. -So ``task_avail_delay`` is the maximum time that the any host will wait to +So ``task_avail_delay`` is the maximum time that any host will wait to take a task. -This means hosts with higher available capacity will be more likely to claim a +Hosts with higher available capacity will be more likely to claim a task, resulting in better utilization of the highest capacity hosts. @@ -206,6 +186,20 @@ This causes an error in python3 if ``SafeConfigParser`` is used, so we use ``RawConfigParser`` instead. +**Handle bare merge mode** + +| PR: https://pagure.io/koji/pull-request/1411 +| PR: https://pagure.io/koji/pull-request/1516 +| PR: https://pagure.io/koji/pull-request/1502 + + +This feature adds a new merge mode for external repos named ``bare``. +This mode is intended for use with modularity. + +Use of this mode requires createrepo_c version 0.14.0 or later on the builders +that handle the createrepo tasks. + + System Changes @@ -270,7 +264,7 @@ This feature adds new policy tests to match ``version`` and ``release``. This tests are glob pattern matches. -**New 'buildtype' test for policies** +**Allow hub policy to match build type** | PR: https://pagure.io/koji/pull-request/1415 @@ -299,6 +293,8 @@ separate permissions: ``target`` This permission governs adding, editing, and deleting targets. +Koji administrators may want to consider reducing the number of users with +full ``admin`` permission. **Option to generate separate source repo** From 257d3a1a7b200ffb4e67f2879425b242819b531c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 7/14] migration note about permissions Fixes: https://pagure.io/koji/issue/1548 --- diff --git a/docs/source/migrating_to_1.18.rst b/docs/source/migrating_to_1.18.rst index 353046c..99736cc 100644 --- a/docs/source/migrating_to_1.18.rst +++ b/docs/source/migrating_to_1.18.rst @@ -25,6 +25,15 @@ database. # psql koji koji Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 8/14] update version for PyPi --- diff --git a/setup.py b/setup.py index e5916ad..69c8555 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def get_install_requires(): setup( name="koji", - version="1.17.0", + version="1.18.0", description=("Koji is a system for building and tracking RPMS. The base" " package contains shared libraries and the command-line" " interface."), From 5d8037c934f96122fabb21d1fa8fe76032b65d21 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 9/14] typo --- diff --git a/docs/source/release_notes_1.18.rst b/docs/source/release_notes_1.18.rst index d27f218..e7e07df 100644 --- a/docs/source/release_notes_1.18.rst +++ b/docs/source/release_notes_1.18.rst @@ -126,7 +126,7 @@ timestamps. Builder Changes --------------- -**Rebuild SPMS before building** +**Rebuild SRPMS before building** | PR: https://pagure.io/koji/pull-request/1462 From a7c9b11bc9aaf50bed04a4ad71e4e3ba83bfd531 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 10/14] update changelog --- diff --git a/koji.spec b/koji.spec index 75b3548..4d23ed1 100644 --- a/koji.spec +++ b/koji.spec @@ -714,6 +714,9 @@ fi %changelog * Fri Aug 2 2019 Mike McLean - 1.18.0-1 +- PR#1555: use _writeInheritanceData in _create_tag +- PR#1580: cli: verify user in block-notification command +- PR#1578: cli:fix typo in mock-config - PR#1464: API for reserving NVRs for content generators - PR#898: Add support for tag/target macros for Mageia - PR#1544: use RawConfigParser for kojid From 15e544fc34bfd4e4fdf508e052124fffb40f8810 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 11/14] update changelog again --- diff --git a/koji.spec b/koji.spec index 4d23ed1..97a59fe 100644 --- a/koji.spec +++ b/koji.spec @@ -714,6 +714,7 @@ fi %changelog * Fri Aug 2 2019 Mike McLean - 1.18.0-1 +- PR#1591: secrets import is missing 'else' variant - PR#1555: use _writeInheritanceData in _create_tag - PR#1580: cli: verify user in block-notification command - PR#1578: cli:fix typo in mock-config From 4dc84bbe8739f04a4c497f67295df27d014631f9 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 12/14] another changelog update --- diff --git a/koji.spec b/koji.spec index 97a59fe..6cf8be1 100644 --- a/koji.spec +++ b/koji.spec @@ -714,6 +714,10 @@ fi %changelog * Fri Aug 2 2019 Mike McLean - 1.18.0-1 +- PR#1606: pull owner from correct place +- PR#1602: copy updated policy for reserved cg builds +- PR#1601: fix recycling build due to cg +- PR#1597: Backward-compatible fix for CG import - PR#1591: secrets import is missing 'else' variant - PR#1555: use _writeInheritanceData in _create_tag - PR#1580: cli: verify user in block-notification command From 57ebae64e63340e53628407d7257e465e3f1568b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:31 +0000 Subject: [PATCH 13/14] include a few more PRs in release notes --- diff --git a/docs/source/release_notes_1.18.rst b/docs/source/release_notes_1.18.rst index e7e07df..cd1d15e 100644 --- a/docs/source/release_notes_1.18.rst +++ b/docs/source/release_notes_1.18.rst @@ -209,6 +209,10 @@ System Changes **API for reserving NVRs for content generators** | PR: https://pagure.io/koji/pull-request/1464 +| PR: https://pagure.io/koji/pull-request/1597 +| PR: https://pagure.io/koji/pull-request/1601 +| PR: https://pagure.io/koji/pull-request/1602 +| PR: https://pagure.io/koji/pull-request/1606 This feature allows content generators to reserve NVRs earlier in the build process similar to builds performed by ``kojid``. The NVR is reserved by @@ -235,6 +239,7 @@ For example, to set the dist tag for a given tag, you could use a command like: **Per-tag configuration for module_hotfixes setting** | PR: https://pagure.io/koji/pull-request/1524 +| PR: https://pagure.io/koji/pull-request/1578 Koji now handles the field ``mock.yum.module_hotfixes`` in the tag extra. When set, kojid will set ``module_hotfixes=0/1`` in the yum portion of the @@ -244,6 +249,7 @@ mock configuration for a buildroot. **Allow users to opt out of notifications** | PR: https://pagure.io/koji/pull-request/1417 +| PR: https://pagure.io/koji/pull-request/1580 This feature lets users opt out of notifications that they would otherwise automatically recieve, such as build and tag notifications for: From e07808f7bf2a9fbb3cc35a7bf532875608a1c9ef Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 09 2019 14:12:56 +0000 Subject: [PATCH 14/14] fix release date --- diff --git a/koji.spec b/koji.spec index 6cf8be1..05eefe3 100644 --- a/koji.spec +++ b/koji.spec @@ -713,7 +713,7 @@ fi %endif %changelog -* Fri Aug 2 2019 Mike McLean - 1.18.0-1 +* Fri Aug 9 2019 Mike McLean - 1.18.0-1 - PR#1606: pull owner from correct place - PR#1602: copy updated policy for reserved cg builds - PR#1601: fix recycling build due to cg