From 54b15ce31a12ce1292637856fff9d0ca15370132 Mon Sep 17 00:00:00 2001 From: Aron Parsons Date: Jun 05 2020 08:20:30 +0000 Subject: [PATCH 1/5] builder: add bootstrap-chroot support for mock fixes https://pagure.io/koji/issue/1017 --- diff --git a/builder/kojid b/builder/kojid index d450e05..34c9699 100755 --- a/builder/kojid +++ b/builder/kojid @@ -548,7 +548,13 @@ class BuildRoot(object): return koji.pathinfo.taskrelpath(self.task_id) def init(self): - rv = self.mock(['--init']) + args = ['--init'] + + if 'mock.bootstrap_chroot' in self.config['extra']: + if self.config['extra']['mock.bootstrap_chroot']: + args.append('--bootstrap-chroot') + + rv = self.mock(args) if rv: self.expire() diff --git a/docs/source/using_the_koji_build_system.rst b/docs/source/using_the_koji_build_system.rst index 92d1db0..953e0d3 100644 --- a/docs/source/using_the_koji_build_system.rst +++ b/docs/source/using_the_koji_build_system.rst @@ -394,6 +394,9 @@ environment follows: * `mock.new_chroot` - 0/1 value. If it is set, `--new-chroot` or `--old-chroot` option is appended to any mock call. If it is not set, mock's default behavior is used. +* `mock.bootstrap_chroot` - 0/1 value. If it is set, `--bootstrap-chroot` + is appended to the mock init call. This tells mock to build in two stages, + using chroot rpm for creating the build chroot You may also specify per-tag environment variables for mock to use. For example, to set the CC environment variable to clang, you could From 34924847d1aa9de1201e6b42d142bd827139b836 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 05 2020 08:21:13 +0000 Subject: [PATCH 2/5] extend to allow also boostrap-image Fixes: https://pagure.io/koji/issue/1017 --- diff --git a/builder/kojid b/builder/kojid index 34c9699..4ba6838 100755 --- a/builder/kojid +++ b/builder/kojid @@ -300,6 +300,10 @@ class BuildRoot(object): opts['tag_macros'][macro] = self.config['extra'][key] elif key.startswith('rpm.env.'): opts['tag_envvars'][key[8:]] = self.config['extra'][key] + if 'mock.bootstrap_chroot' in self.config['extra']: + opts['bootstrap_chroot'] = bool(self.config['extra']['mock.bootstrap_chroot']) + opts['bootstrap_image'] = self.config['extra'].get('mock.bootstrap_image') + output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) # write config @@ -548,13 +552,7 @@ class BuildRoot(object): return koji.pathinfo.taskrelpath(self.task_id) def init(self): - args = ['--init'] - - if 'mock.bootstrap_chroot' in self.config['extra']: - if self.config['extra']['mock.bootstrap_chroot']: - args.append('--bootstrap-chroot') - - rv = self.mock(args) + rv = self.mock(['--init']) if rv: self.expire() diff --git a/docs/source/using_the_koji_build_system.rst b/docs/source/using_the_koji_build_system.rst index 953e0d3..52dd1b1 100644 --- a/docs/source/using_the_koji_build_system.rst +++ b/docs/source/using_the_koji_build_system.rst @@ -389,14 +389,44 @@ environment follows: koji edit-tag dnf-fedora-tag -x mock.package_manager=dnf -* `mock.package_manager` - If this is set, it will override mock's default - package manager. Typically used with `yum` or `dnf` values. -* `mock.new_chroot` - 0/1 value. If it is set, `--new-chroot` or +* ``mock.package_manager`` - If this is set, it will override mock's default + package manager. Typically used with ``yum`` or ``dnf`` values. +* ``mock.new_chroot`` - 0/1 value. If it is set, `--new-chroot` or `--old-chroot` option is appended to any mock call. If it is not set, mock's default behavior is used. -* `mock.bootstrap_chroot` - 0/1 value. If it is set, `--bootstrap-chroot` +* ``mock.bootstrap_chroot`` - 0/1 value. If it is set, ``--bootstrap-chroot`` is appended to the mock init call. This tells mock to build in two stages, - using chroot rpm for creating the build chroot + using chroot rpm for creating the build chroot. If it is not set, mock's + default behaviour is used. (Note, that it changed in mock `1.4.1 + `_. + Note, that it is not turn on by default by koji, as it is often not needed and + it consumes additional resources (larger buildroot, downloading more data). +* ``mock.bootstrap_image`` - set to name of image, which can builder's podman + download (e.g. ``fedora:32``). See mock's `doc + `_ + before using this. You could need it, but do it with following + recommendations: + + - you need to have builders with `podman `_ installed and + working. + + - use concrete hashes not potentially moving tags. Otherwise, you can get into + harder debugging and auditing. + + - builders can consume space during time, no cleanup is made for podman's + image cache. So, you'll probably want to run something like ``podman rmi + `podman images -a --quiet``` periodically via cron or use some other + cache-cleaning mechanism. Even simple task will consume roughly three times + more space than without bootstrap image (downloaded image + exploded + boostrap dir + mock's buildroot itself) + + - be sure, that your podman is configured properly and it downloads images + only from trusted sources. Note, that this setting effectivelly circumvents + network isolation *inside* buildroot, as outside DNS, etc. can be spoofed. + + - this option will automatically turn ``mock.bootstrap_chroot`` (this is how + it is implemented in mock) + You may also specify per-tag environment variables for mock to use. For example, to set the CC environment variable to clang, you could diff --git a/koji/__init__.py b/koji/__init__.py index 4dc8c7d..717cd7d 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1584,6 +1584,11 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) } if opts.get('package_manager'): config_opts['package_manager'] = opts['package_manager'] + if opts.get('bootstrap_image'): + config_opts['use_bootstrap_image'] = True + config_opts['bootstrap_image'] = opts['bootstrap_image'] + if 'bootstrap_chroot' in opts: + config_opts['bootstrap_chroot'] = opts['bootstrap_chroot'] # bind_opts are used to mount parts (or all of) /dev if needed. # See kojid::LiveCDTask for a look at this option in action. From 455a997d9d848f843b0a40d9988a6f6d0fb746b9 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 05 2020 08:21:18 +0000 Subject: [PATCH 3/5] use global option to enable bootstrap-image --- diff --git a/builder/kojid b/builder/kojid index 4ba6838..681c2b0 100755 --- a/builder/kojid +++ b/builder/kojid @@ -302,7 +302,10 @@ class BuildRoot(object): opts['tag_envvars'][key[8:]] = self.config['extra'][key] if 'mock.bootstrap_chroot' in self.config['extra']: opts['bootstrap_chroot'] = bool(self.config['extra']['mock.bootstrap_chroot']) - opts['bootstrap_image'] = self.config['extra'].get('mock.bootstrap_image') + + # it must be allowed in kojid.conf *and* in tag's extra info + opts['bootstrap_image'] = self.options.mock_boostrap_image and \ + self.config['extra'].get('mock.bootstrap_image') output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) @@ -6401,6 +6404,7 @@ def get_options(): 'use_createrepo_c': True, 'createrepo_skip_stat': True, 'createrepo_update': True, + 'mock_boostrap_image': False, 'pkgurl': None, 'allowed_scms': '', 'scm_credentials_dir': None, diff --git a/builder/kojid.conf b/builder/kojid.conf index 892c50a..5f5ec28 100644 --- a/builder/kojid.conf +++ b/builder/kojid.conf @@ -116,6 +116,9 @@ from_addr=Koji Build System ;if set to True, failing subtask will not automatically cancel other siblings ;build_arch_can_fail = False +;if set to True, tag extra 'mock.bootstrap_image' can be used +;mock_boostrap_image = False + ;image build with raw-xz type will use following xz options ;xz_options=-z6T0 diff --git a/docs/source/using_the_koji_build_system.rst b/docs/source/using_the_koji_build_system.rst index 52dd1b1..279e9cb 100644 --- a/docs/source/using_the_koji_build_system.rst +++ b/docs/source/using_the_koji_build_system.rst @@ -407,6 +407,9 @@ environment follows: before using this. You could need it, but do it with following recommendations: + - you need to explicitly allow builders to do that (``mock_boostrap_image = + True`` in ``kojid.conf``). + - you need to have builders with `podman `_ installed and working. From 27e0d3cbf7b837d9684984eb747ea57e1f5ce03c Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 05 2020 08:21:44 +0000 Subject: [PATCH 4/5] fix option name --- diff --git a/builder/kojid b/builder/kojid index 681c2b0..9ec07f3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -300,11 +300,11 @@ class BuildRoot(object): opts['tag_macros'][macro] = self.config['extra'][key] elif key.startswith('rpm.env.'): opts['tag_envvars'][key[8:]] = self.config['extra'][key] - if 'mock.bootstrap_chroot' in self.config['extra']: - opts['bootstrap_chroot'] = bool(self.config['extra']['mock.bootstrap_chroot']) + if 'mock.use_bootstrap' in self.config['extra']: + opts['use_bootstrap'] = bool(self.config['extra']['mock.use_bootstrap']) # it must be allowed in kojid.conf *and* in tag's extra info - opts['bootstrap_image'] = self.options.mock_boostrap_image and \ + opts['bootstrap_image'] = self.options.mock_bootstrap_image and \ self.config['extra'].get('mock.bootstrap_image') output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) @@ -6404,7 +6404,7 @@ def get_options(): 'use_createrepo_c': True, 'createrepo_skip_stat': True, 'createrepo_update': True, - 'mock_boostrap_image': False, + 'mock_bootstrap_image': False, 'pkgurl': None, 'allowed_scms': '', 'scm_credentials_dir': None, diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 2385862..82df13e 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -1024,7 +1024,11 @@ def anon_handle_mock_config(goptions, session, args): opts['package_manager'] = buildcfg['extra']['mock.package_manager'] if 'mock.yum.module_hotfixes' in buildcfg['extra']: opts['module_hotfixes'] = buildcfg['extra']['mock.yum.module_hotfixes'] - + if 'mock.bootstrap_image' in buildcfg['extra']: + opts['use_bootstrap_image'] = True + opts['bootstrap_image'] = buildcfg['extra']['mock.bootstrap_image'] + if 'mock.use_bootstrap' in buildcfg['extra']: + opts['use_bootstrap'] = buildcfg['extra']['mock.use_bootstrap'] output = koji.genMockConfig(name, arch, **opts) if options.ofile: with open(options.ofile, 'w') as fo: diff --git a/docs/source/using_the_koji_build_system.rst b/docs/source/using_the_koji_build_system.rst index 279e9cb..e7b5327 100644 --- a/docs/source/using_the_koji_build_system.rst +++ b/docs/source/using_the_koji_build_system.rst @@ -394,7 +394,7 @@ environment follows: * ``mock.new_chroot`` - 0/1 value. If it is set, `--new-chroot` or `--old-chroot` option is appended to any mock call. If it is not set, mock's default behavior is used. -* ``mock.bootstrap_chroot`` - 0/1 value. If it is set, ``--bootstrap-chroot`` +* ``mock.use_bootstrap`` - 0/1 value. If it is set, ``--bootstrap-chroot`` is appended to the mock init call. This tells mock to build in two stages, using chroot rpm for creating the build chroot. If it is not set, mock's default behaviour is used. (Note, that it changed in mock `1.4.1 @@ -407,7 +407,7 @@ environment follows: before using this. You could need it, but do it with following recommendations: - - you need to explicitly allow builders to do that (``mock_boostrap_image = + - you need to explicitly allow builders to do that (``mock_bootstrap_image = True`` in ``kojid.conf``). - you need to have builders with `podman `_ installed and @@ -421,13 +421,13 @@ environment follows: `podman images -a --quiet``` periodically via cron or use some other cache-cleaning mechanism. Even simple task will consume roughly three times more space than without bootstrap image (downloaded image + exploded - boostrap dir + mock's buildroot itself) + bootstrap dir + mock's buildroot itself) - be sure, that your podman is configured properly and it downloads images only from trusted sources. Note, that this setting effectivelly circumvents network isolation *inside* buildroot, as outside DNS, etc. can be spoofed. - - this option will automatically turn ``mock.bootstrap_chroot`` (this is how + - this option will automatically turn ``mock.use_bootstrap`` (this is how it is implemented in mock) diff --git a/koji/__init__.py b/koji/__init__.py index 717cd7d..c2dca0c 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1587,8 +1587,8 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) if opts.get('bootstrap_image'): config_opts['use_bootstrap_image'] = True config_opts['bootstrap_image'] = opts['bootstrap_image'] - if 'bootstrap_chroot' in opts: - config_opts['bootstrap_chroot'] = opts['bootstrap_chroot'] + if 'use_bootstrap' in opts: + config_opts['use_bootstrap'] = bool(opts['use_bootstrap']) # bind_opts are used to mount parts (or all of) /dev if needed. # See kojid::LiveCDTask for a look at this option in action. From ac724ba50f86fcd870e7005c430fb0109bac735a Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 05 2020 08:21:46 +0000 Subject: [PATCH 5/5] warning for disabled buildroot_image --- diff --git a/builder/kojid b/builder/kojid index 9ec07f3..71408f9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -304,6 +304,9 @@ class BuildRoot(object): opts['use_bootstrap'] = bool(self.config['extra']['mock.use_bootstrap']) # it must be allowed in kojid.conf *and* in tag's extra info + if not self.options.mock_bootstrap_image and \ + self.config['extra'].get('mock.bootstrap_image'): + self.logger.warning("Mock bootstrap image requested by buildroot %d, but forbidden on builder" % self.id) opts['bootstrap_image'] = self.options.mock_bootstrap_image and \ self.config['extra'].get('mock.bootstrap_image')